Add Parallel Pod Management Policy to standalone pgadmin StatefulSet.

Author: dsessler7

internal/controller/standalone_pgadmin/statefulset.go

@@ -19,7 +19,9 @@ import (
 
 appsv1 "k8s.io/api/apps/v1"
 corev1 "k8s.io/api/core/v1"
+apierrors "k8s.io/apimachinery/pkg/api/errors"
 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+"sigs.k8s.io/controller-runtime/pkg/client"
 
 "github.com/pkg/errors"
 
@@ -34,6 +36,30 @@ func (r *PGAdminReconciler) reconcilePGAdminStatefulSet(
 configmap *corev1.ConfigMap, dataVolume *corev1.PersistentVolumeClaim,
 ) error {
 sts := statefulset(r, pgadmin, configmap, dataVolume)
+
+// Previous versions of PGO used a StatefulSet Pod Management Policy that could leave the Pod
+// in a failed state. When we see that it has the wrong policy, we will delete the StatefulSet
+// and then recreate it with the correct policy, as this is not a property that can be patched.
+// When we delete the StatefulSet, we will leave its Pods in place. They will be claimed by
+// the StatefulSet that gets created in the next reconcile.
+existing := &appsv1.StatefulSet{}
+if err := errors.WithStack(r.Client.Get(ctx, client.ObjectKeyFromObject(sts), existing)); err != nil {
+if !apierrors.IsNotFound(err) {
+return err
+}
+} else {
+if existing.Spec.PodManagementPolicy != sts.Spec.PodManagementPolicy {
+// We want to delete the STS without affecting the Pods, so we set the PropagationPolicy to Orphan.
+// The orphaned Pods will be claimed by the StatefulSet that will be created in the next reconcile.
+uid := existing.GetUID()
+version := existing.GetResourceVersion()
+exactly := client.Preconditions{UID: &uid, ResourceVersion: &version}
+propagate := client.PropagationPolicy(metav1.DeletePropagationOrphan)
+
+return errors.WithStack(client.IgnoreNotFound(r.Client.Delete(ctx, existing, exactly, propagate)))
+}
+}
+
 if err := errors.WithStack(r.setControllerReference(pgadmin, sts)); err != nil {
 return err
 }
@@ -70,15 +96,13 @@ func statefulset(
 // Don't clutter the namespace with extra ControllerRevisions.
 sts.Spec.RevisionHistoryLimit = initialize.Int32(0)
 
-// Set the StatefulSet update strategy to "RollingUpdate", and the Partition size for the
-// update strategy to 0 (note that these are the defaults for a StatefulSet). This means
-// every pod of the StatefulSet will be deleted and recreated when the Pod template changes.
-// - https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#rolling-updates
-// - https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#forced-rollback
+// Use StatefulSet's "RollingUpdate" strategy and "Parallel" policy to roll
+// out changes to pods even when not Running or not Ready.
+// - https://docs.k8s.io/concepts/workloads/controllers/statefulset/#rolling-updates
+// - https://docs.k8s.io/concepts/workloads/controllers/statefulset/#forced-rollback
+// - https://kep.k8s.io/3541
+sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
 sts.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
-sts.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateStatefulSetStrategy{
-Partition: initialize.Int32(0),
-}
 
 // Use scheduling constraints from the cluster spec.
 sts.Spec.Template.Spec.Affinity = pgadmin.Spec.Affinity