Doc

CONTROLLER_VERSION

@@ -1 +1 @@
-1.29.35
+1.29.36

pkg/controller/clusterstate/cache/cache.go

@@ -305,21 +305,21 @@ func (sc *ClusterStateCache) updateState() error {
 }
 
 klog.V(8).Infof("Total capacity %+v, used %+v, free space %+v", total, used, idle)
-if klog.V(10).Enabled() {
+if klog.V(12).Enabled() {
 // CPU histogram
 metricCPU := &dto.Metric{}
 (*newIdleHistogram.MilliCPU).Write(metricCPU)
-klog.V(10).Infof("[updateState] CPU histogram:\n%s", proto.MarshalTextString(metricCPU))
+klog.V(12).Infof("[updateState] CPU histogram:\n%s", proto.MarshalTextString(metricCPU))
 
 // Memory histogram
 metricMem := &dto.Metric{}
 (*newIdleHistogram.Memory).Write(metricMem)
-klog.V(10).Infof("[updateState] Memory histogram:\n%s", proto.MarshalTextString(metricMem))
+klog.V(12).Infof("[updateState] Memory histogram:\n%s", proto.MarshalTextString(metricMem))
 
 // GPU histogram
 metricGPU := &dto.Metric{}
 (*newIdleHistogram.GPU).Write(metricGPU)
-klog.V(10).Infof("[updateState] GPU histogram:\n%s", proto.MarshalTextString(metricGPU))
+klog.V(12).Infof("[updateState] GPU histogram:\n%s", proto.MarshalTextString(metricGPU))
 }
 
 err := sc.saveState(idle, total, newIdleHistogram)

pkg/controller/queuejob/queuejob_controller_ex.go

@@ -1599,6 +1599,16 @@ func (cc *XController) manageQueueJob(qj *arbv1.AppWrapper, podPhaseChanges bool
 return nil
 } // End of first execution of qj to add to qjqueue for ScheduleNext
 
+//Handle recovery condition
+if !qj.Status.CanRun && qj.Status.State == arbv1.AppWrapperStateEnqueued &&
+!cc.qjqueue.IfExistUnschedulableQ(qj) && !cc.qjqueue.IfExistActiveQ(qj){
+cc.qjqueue.AddIfNotPresent(qj)
+klog.V(3).Infof("[manageQueueJob] Recovered AppWrapper %s%s - added to active queue, Status=%+v",
+qj.Namespace, qj.Name, qj.Status)
+
+return nil
+}
+
 // add qj to Etcd for dispatch
 if qj.Status.CanRun && qj.Status.State != arbv1.AppWrapperStateActive {
 qj.Status.State = arbv1.AppWrapperStateActive