Read res req from custom pod res

CONTROLLER_VERSION

@@ -1 +1 @@
-1.29.41
+1.29.42

pkg/controller/queuejobresources/genericresource/genericresource.go

@@ -380,21 +380,25 @@ func GetResources(awr *arbv1.AppWrapperGenericResource) (resource *clusterstatea
 var err error
 err = nil
 if awr.GenericTemplate.Raw != nil {
+if len(awr.CustomPodResources) > 0 {
+podresources := awr.CustomPodResources
+for _, item := range podresources {
+res := getPodResources(item)
+totalresource = totalresource.Add(res)
+}
+klog.V(4).Infof("[GetResources] Requested total allocation resource from custompodresources `%v`.\n", totalresource)
+return totalresource, err
+}
 hasContainer, replicas, containers := hasFields(awr.GenericTemplate)
 if hasContainer {
 for _, item := range containers {
 res := getContainerResources(item, replicas)
 totalresource = totalresource.Add(res)
 }
-klog.V(8).Infof("[GetResources] Requested total allocation resource from containers `%v`.\n", totalresource)
-} else {
-podresources := awr.CustomPodResources
-for _, item := range podresources {
-res := getPodResources(item)
-totalresource = totalresource.Add(res)
-}
-klog.V(8).Infof("[GetResources] Requested total allocation resource from pods `%v`.\n", totalresource)
+klog.V(4).Infof("[GetResources] Requested total allocation resource from containers `%v`.\n", totalresource)
+return totalresource, err
 }
+
 } else {
 err = fmt.Errorf("generic template raw object is not defined (nil)")
 }

test/e2e/queue.go

@@ -284,6 +284,79 @@ var _ = Describe("AppWrapper E2E Test", func() {
 
 })
 
+It("MCAD Custom Pod Resources Test", func() {
+fmt.Fprintf(os.Stdout, "[e2e] MCAD Custom Pod Resources Test - Started.\n")
+context := initTestContext()
+var appwrappers []*arbv1.AppWrapper
+appwrappersPtr := &appwrappers
+defer cleanupTestObjectsPtr(context, appwrappersPtr)
+
+// This should fit on cluster with customPodResources matching deployment resource demands so AW pods are created
+aw := createGenericDeploymentCustomPodResourcesWithCPUAW(
+context, "aw-deployment-2-550-vs-550-cpu", "550m", "550m", 2)
+
+appwrappers = append(appwrappers, aw)
+
+err := waitAWAnyPodsExists(context, aw)
+Expect(err).NotTo(HaveOccurred())
+
+err = waitAWPodsReady(context, aw)
+Expect(err).NotTo(HaveOccurred())
+})
+
+
+It("MCAD Bad Custom Pod Resources vs. Deployment Pod Resource Not Queuing Test", func() {
+fmt.Fprintf(os.Stdout, "[e2e] MCAD Bad Custom Pod Resources vs. Deployment Pod Resource Not Queuing Test - Started.\n")
+context := initTestContext()
+var appwrappers []*arbv1.AppWrapper
+appwrappersPtr := &appwrappers
+defer cleanupTestObjectsPtr(context, appwrappersPtr)
+
+// This should fill up the worker node and most of the master node
+aw := createDeploymentAWwith550CPU(context, "aw-deployment-2-550cpu")
+appwrappers = append(appwrappers, aw)
+
+err := waitAWPodsReady(context, aw)
+Expect(err).NotTo(HaveOccurred())
+
+// This should not fit on cluster but customPodResources is incorrect so AW pods are created
+aw2 := createGenericDeploymentCustomPodResourcesWithCPUAW(
+context, "aw-deployment-2-425-vs-426-cpu", "425m", "426m", 2)
+
+appwrappers = append(appwrappers, aw2)
+
+err = waitAWAnyPodsExists(context, aw2)
+Expect(err).NotTo(HaveOccurred())
+
+err = waitAWPodsReady(context, aw2)
+Expect(err).To(HaveOccurred())
+})
+
+It("MCAD Bad Custom Pod Resources vs. Deployment Pod Resource Queuing Test 2", func() {
+fmt.Fprintf(os.Stdout, "[e2e] MCAD Bad Custom Pod Resources vs. Deployment Pod Resource Queuing Test 2 - Started.\n")
+context := initTestContext()
+var appwrappers []*arbv1.AppWrapper
+appwrappersPtr := &appwrappers
+defer cleanupTestObjectsPtr(context, appwrappersPtr)
+
+// This should fill up the worker node and most of the master node
+aw := createDeploymentAWwith550CPU(context, "aw-deployment-2-550cpu")
+appwrappers = append(appwrappers, aw)
+
+err := waitAWPodsReady(context, aw)
+Expect(err).NotTo(HaveOccurred())
+
+// This should fit on cluster but customPodResources is incorrect so AW pods are not created
+aw2 := createGenericDeploymentCustomPodResourcesWithCPUAW(
+context, "aw-deployment-2-426-vs-425-cpu", "426m", "425m", 2)
+
+appwrappers = append(appwrappers, aw2)
+
+err = waitAWAnyPodsExists(context, aw2)
+Expect(err).To(HaveOccurred())
+
+})
+
 It("MCAD CPU Accounting Queuing Test", func() {
 fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Accounting Queuing Test - Started.\n")
 context := initTestContext()

test/e2e/util.go

@@ -1592,6 +1592,94 @@ func createGenericDeploymentWithCPUAW(context *context, name string, cpuDemand s
 return appwrapper
 }
 
+func createGenericDeploymentCustomPodResourcesWithCPUAW(context *context, name string, customPodCpuDemand string, cpuDemand string, replicas int) *arbv1.AppWrapper {
+rb := []byte(fmt.Sprintf(`{
+"apiVersion": "apps/v1",
+"kind": "Deployment", 
+"metadata": {
+"name": "%s",
+"namespace": "test",
+"labels": {
+"app": "%s"
+}
+},
+"spec": {
+"replicas": %d,
+"selector": {
+"matchLabels": {
+"app": "%s"
+}
+},
+"template": {
+"metadata": {
+"labels": {
+"app": "%s"
+},
+"annotations": {
+"appwrapper.mcad.ibm.com/appwrapper-name": "%s"
+}
+},
+"spec": {
+"containers": [
+{
+"name": "%s",
+"image": "k8s.gcr.io/echoserver:1.4",
+"resources": {
+"requests": {
+"cpu": "%s"
+}
+},
+"ports": [
+{
+"containerPort": 80
+}
+]
+}
+]
+}
+}
+}} `, name, name, replicas, name, name, name, name, cpuDemand))
+
+var schedSpecMin int = replicas
+var customCpuResource = v1.ResourceList{"cpu": resource.MustParse(customPodCpuDemand)}
+
+aw := &arbv1.AppWrapper{
+ObjectMeta: metav1.ObjectMeta{
+Name: name,
+Namespace: context.namespace,
+},
+Spec: arbv1.AppWrapperSpec{
+SchedSpec: arbv1.SchedulingSpecTemplate{
+MinAvailable: schedSpecMin,
+},
+AggrResources: arbv1.AppWrapperResourceList{
+GenericItems: []arbv1.AppWrapperGenericResource{
+{
+ObjectMeta: metav1.ObjectMeta{
+Name: fmt.Sprintf("%s-%s", name, "item1"),
+Namespace: context.namespace,
+},
+CustomPodResources: []arbv1.CustomPodResourceTemplate{
+{
+Replicas: replicas,
+Requests: customCpuResource,
+},
+},
+GenericTemplate: runtime.RawExtension{
+Raw: rb,
+},
+},
+},
+},
+},
+}
+
+appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw)
+Expect(err).NotTo(HaveOccurred())
+
+return appwrapper
+}
+
 func createNamespaceAW(context *context, name string) *arbv1.AppWrapper {
 rb := []byte(`{"apiVersion": "v1",
 "kind": "Namespace",