⚡️ Improve scale-up/down performance by processing pod-template changes before replicas (#1310)

Author: andersjohnsen

go.mod

@@ -32,7 +32,7 @@ require (
 github.com/nyaruka/phonenumbers v1.1.7
 github.com/pkg/errors v0.9.1
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.70.0
-github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f
+github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716
 github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f
 github.com/rivo/tview v0.0.0-20240524063012-037df494fb76
 github.com/robfig/cron v1.2.0

go.sum

@@ -309,8 +309,8 @@ github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJ
 github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f h1:D87qhahpJla10fB/Ptd9/Ocg5ktGFJ3HosQDBrnn9vk=
-github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f/go.mod h1:0yAwJnPID3gHfB3ETQAFYGM1tkNWkJKbVzauZiHmHKw=
+github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716 h1:QpieTeTr+uVcjBcQJuGpLz7sFuHl87EKf0lST7gK9qc=
+github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716/go.mod h1:0yAwJnPID3gHfB3ETQAFYGM1tkNWkJKbVzauZiHmHKw=
 github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f h1:SScDJVwFt/QzURstwUdi/ejnRoqB1ZsBLlICPD6Lt0c=
 github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f/go.mod h1:DJAwoPmFI6Jo71n/2VfFk/Nyzlyd24ARWCLC4Hsm1gg=
 github.com/rivo/tview v0.0.0-20240524063012-037df494fb76 h1:iqvDlgyjmqleATtFbA7c14djmPh2n4mCYUv7JlD/ruA=

pkg/pipeline/request.go

@@ -505,7 +505,7 @@ func (r *RequestBase) applyChange(ctx context.Context, key ObjectKey, state Reso
 switch state {
 case ResourceStateUpdated:
 r.logger.Info("update object", "object", key)
-obj := r.newObjects[key]
+object := r.newObjects[key]
 
 // Edge case: Deployments, when they are scaled, will apply changes in the following order:
 // 1) Change the number of replicas (pods) in the current ReplicaSet
@@ -518,18 +518,33 @@ func (r *RequestBase) applyChange(ctx context.Context, key ObjectKey, state Reso
 // 1) First, we apply the template changes (use current number of replicas)
 // 2) Then apply the scale changes.
 // That way the scale is only being applied to the new ReplicaSet.
-if key.GroupVersionKind == AppsDeploymentGVK && obj.Materialized != nil {
-currentObj := obj.Current.(*v1.Deployment)
-newObj := obj.New.(*v1.Deployment)
-materializedObj := obj.Materialized.(*v1.Deployment)
+if key.GroupVersionKind == AppsDeploymentGVK && object.Materialized != nil {
+currentObj := object.Current.(*v1.Deployment)
+newObj := object.New.(*v1.Deployment)
+materializedObj := object.Materialized.(*v1.Deployment)
+for _, c := range currentObj.Status.Conditions {
+if c.Type == v1.DeploymentProgressing && c.Reason == "ReplicaSetUpdated" {
+// We have an ongoing rollout, wait a little with scale-up.
+if newObj.Spec.Replicas != nil &&
+currentObj.Spec.Replicas != nil &&
+*newObj.Spec.Replicas > *currentObj.Spec.Replicas {
+newObj.Spec.Replicas = currentObj.Spec.Replicas
+}
+}
+}
+
 if !equality.Semantic.DeepEqual(currentObj.Spec.Template, materializedObj.Spec.Template) {
 // We have changes to the template - don't apply potential replica changes yet.
-newObj.Spec.Replicas = currentObj.Spec.Replicas
+if newObj.Spec.Replicas != nil &&
+currentObj.Spec.Replicas != nil &&
+*newObj.Spec.Replicas > *currentObj.Spec.Replicas {
+newObj.Spec.Replicas = currentObj.Spec.Replicas
+}
 }
 }
 
-obj.New.SetResourceVersion(obj.Current.GetResourceVersion())
-if err := r.client.Update(ctx, obj.New); err != nil {
+object.New.SetResourceVersion(object.Current.GetResourceVersion())
+if err := r.client.Update(ctx, object.New); err != nil {
 return fmt.Errorf("could not update %s: %w", key.GroupVersionKind, err)
 }