add event recording and ingress info in error messages (kubernetes-sigs#1637)

Author: M00nF1sh

controllers/elbv2/targetgroupbinding_controller.go

@@ -18,9 +18,11 @@ package controllers
 
 import (
 "context"
+"fmt"
 "github.com/aws/aws-sdk-go/aws"
 "github.com/pkg/errors"
 corev1 "k8s.io/api/core/v1"
+"k8s.io/client-go/tools/record"
 "sigs.k8s.io/aws-load-balancer-controller/controllers/elbv2/eventhandlers"
 "sigs.k8s.io/aws-load-balancer-controller/pkg/config"
 "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
@@ -42,12 +44,13 @@ const (
 )
 
 // NewTargetGroupBindingReconciler constructs new targetGroupBindingReconciler
-func NewTargetGroupBindingReconciler(k8sClient client.Client, finalizerManager k8s.FinalizerManager,
+func NewTargetGroupBindingReconciler(k8sClient client.Client, eventRecorder record.EventRecorder, finalizerManager k8s.FinalizerManager,
 tgbResourceManager targetgroupbinding.ResourceManager, config config.ControllerConfig,
 logger logr.Logger) *targetGroupBindingReconciler {
 
 return &targetGroupBindingReconciler{
 k8sClient: k8sClient,
+eventRecorder: eventRecorder,
 finalizerManager: finalizerManager,
 tgbResourceManager: tgbResourceManager,
 logger: logger,
@@ -59,6 +62,7 @@ func NewTargetGroupBindingReconciler(k8sClient client.Client, finalizerManager k
 // targetGroupBindingReconciler reconciles a TargetGroupBinding object
 type targetGroupBindingReconciler struct {
 k8sClient client.Client
+eventRecorder record.EventRecorder
 finalizerManager k8s.FinalizerManager
 tgbResourceManager targetgroupbinding.ResourceManager
 logger logr.Logger
@@ -95,23 +99,29 @@ func (r *targetGroupBindingReconciler) reconcile(req ctrl.Request) error {
 
 func (r *targetGroupBindingReconciler) reconcileTargetGroupBinding(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error {
 if err := r.finalizerManager.AddFinalizers(ctx, tgb, targetGroupBindingFinalizer); err != nil {
+r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
 return err
 }
 if err := r.tgbResourceManager.Reconcile(ctx, tgb); err != nil {
 return err
 }
 if err := r.updateTargetGroupBindingStatus(ctx, tgb); err != nil {
+r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
 return err
 }
+
+r.eventRecorder.Event(tgb, corev1.EventTypeNormal, k8s.TargetGroupBindingEventReasonSuccessfullyReconciled, "Successfully reconciled")
 return nil
 }
 
 func (r *targetGroupBindingReconciler) cleanupTargetGroupBinding(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error {
 if k8s.HasFinalizer(tgb, targetGroupBindingFinalizer) {
 if err := r.tgbResourceManager.Cleanup(ctx, tgb); err != nil {
+r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedCleanup, fmt.Sprintf("Failed cleanup due to %v", err))
 return err
 }
 if err := r.finalizerManager.RemoveFinalizers(ctx, tgb, targetGroupBindingFinalizer); err != nil {
+r.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
 return err
 }
 }

controllers/ingress/eventhandlers/ingress_events.go

@@ -2,6 +2,7 @@ package eventhandlers
 
 import (
 "context"
+"fmt"
 "github.com/go-logr/logr"
 corev1 "k8s.io/api/core/v1"
 networking "k8s.io/api/networking/v1beta1"
@@ -70,7 +71,7 @@ func (h *enqueueRequestsForIngressEvent) enqueueIfBelongsToGroup(queue workqueue
 for _, ing := range ingList {
 groupID, err := h.groupLoader.FindGroupID(context.Background(), ing)
 if err != nil {
-h.eventRecorder.Eventf(ing, corev1.EventTypeWarning, k8s.IngressEventReasonFailedToLoadGroupID, "failed to load groupID for Ingress due to %w", err)
+h.eventRecorder.Event(ing, corev1.EventTypeWarning, k8s.IngressEventReasonFailedLoadGroupID, fmt.Sprintf("failed load groupID due to %v", err))
 continue
 }
 

controllers/ingress/group_controller.go

@@ -2,6 +2,7 @@ package ingress
 
 import (
 "context"
+"fmt"
 "github.com/go-logr/logr"
 "github.com/pkg/errors"
 corev1 "k8s.io/api/core/v1"
@@ -106,7 +107,9 @@ func (r *groupReconciler) reconcile(req ctrl.Request) error {
 if err != nil {
 return err
 }
+
 if err := r.groupFinalizerManager.AddGroupFinalizer(ctx, ingGroupID, ingGroup.Members...); err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
 return err
 }
 
@@ -120,40 +123,52 @@ func (r *groupReconciler) reconcile(req ctrl.Request) error {
 if err != nil {
 return err
 }
-if err := r.updateIngressesStatus(ctx, lbDNS, ingGroup.Members...); err != nil {
+if err := r.updateIngressGroupStatus(ctx, ingGroup, lbDNS); err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
 return err
 }
 }
 
 if len(ingGroup.InactiveMembers) > 0 {
 if err := r.groupFinalizerManager.RemoveGroupFinalizer(ctx, ingGroupID, ingGroup.InactiveMembers...); err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
 return err
 }
 }
 
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeNormal, k8s.IngressEventReasonSuccessfullyReconciled, "Successfully reconciled")
 return nil
 }
 
 func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) {
 stack, lb, err := r.modelBuilder.Build(ctx, ingGroup)
 if err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
 return nil, nil, err
 }
 stackJSON, err := r.stackMarshaller.Marshal(stack)
 if err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
 return nil, nil, err
 }
 r.logger.Info("successfully built model", "model", stackJSON)
 
 if err := r.stackDeployer.Deploy(ctx, stack); err != nil {
+r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
 return nil, nil, err
 }
 r.logger.Info("successfully deployed model", "ingressGroup", ingGroup.ID)
 return stack, lb, err
 }
 
-func (r *groupReconciler) updateIngressesStatus(ctx context.Context, lbDNS string, ingList ...*networking.Ingress) error {
-for _, ing := range ingList {
+func (r *groupReconciler) recordIngressGroupEvent(_ context.Context, ingGroup ingress.Group, eventType string, reason string, message string) {
+for _, ing := range ingGroup.Members {
+r.eventRecorder.Event(ing, eventType, reason, message)
+}
+}
+
+func (r *groupReconciler) updateIngressGroupStatus(ctx context.Context, ingGroup ingress.Group, lbDNS string) error {
+for _, ing := range ingGroup.Members {
 if err := r.updateIngressStatus(ctx, lbDNS, ing); err != nil {
 return err
 }

controllers/service/service_controller.go

@@ -2,6 +2,7 @@ package service
 
 import (
 "context"
+"fmt"
 "github.com/go-logr/logr"
 "github.com/pkg/errors"
 corev1 "k8s.io/api/core/v1"
@@ -91,12 +92,18 @@ func (r *serviceReconciler) reconcile(req ctrl.Request) error {
 func (r *serviceReconciler) buildAndDeployModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, error) {
 stack, lb, err := r.modelBuilder.Build(ctx, svc)
 if err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
 return nil, nil, err
 }
 stackJSON, err := r.stackMarshaller.Marshal(stack)
+if err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
+return nil, nil, err
+}
 r.logger.Info("successfully built model", "model", stackJSON)
 
 if err = r.stackDeployer.Deploy(ctx, stack); err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
 return nil, nil, err
 }
 r.logger.Info("successfully deployed model", "service", k8s.NamespacedName(svc))
@@ -106,6 +113,7 @@ func (r *serviceReconciler) buildAndDeployModel(ctx context.Context, svc *corev1
 
 func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, svc *corev1.Service) error {
 if err := r.finalizerManager.AddFinalizers(ctx, svc, serviceFinalizer); err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
 return err
 }
 _, lb, err := r.buildAndDeployModel(ctx, svc)
@@ -118,8 +126,10 @@ func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context,
 }
 
 if err = r.updateServiceStatus(ctx, lbDNS, svc); err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
 return err
 }
+r.eventRecorder.Event(svc, corev1.EventTypeNormal, k8s.ServiceEventReasonSuccessfullyReconciled, "Successfully reconciled")
 return nil
 }
 
@@ -130,6 +140,7 @@ func (r *serviceReconciler) cleanupLoadBalancerResources(ctx context.Context, sv
 return err
 }
 if err := r.finalizerManager.RemoveFinalizers(ctx, svc, serviceFinalizer); err != nil {
+r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedRemoveFinalizer, fmt.Sprintf("Failed remove finalizer due to %v", err))
 return err
 }
 }

main.go

@@ -111,7 +111,8 @@ func main() {
 svcReconciler := service.NewServiceReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("service"),
 finalizerManager, sgManager, sgReconciler, subnetResolver,
 controllerCFG, ctrl.Log.WithName("controllers").WithName("service"))
-tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), finalizerManager, tgbResManager,
+tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("targetGroupBinding"),
+finalizerManager, tgbResManager,
 controllerCFG, ctrl.Log.WithName("controllers").WithName("targetGroupBinding"))
 ctx := context.Background()
 if err = ingGroupReconciler.SetupWithManager(ctx, mgr); err != nil {

pkg/ingress/model_build_listener_rules.go

@@ -2,9 +2,10 @@ package ingress
 
 import (
 "context"
-"errors"
+"github.com/pkg/errors"
 "fmt"
 networking "k8s.io/api/networking/v1beta1"
+"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
 elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
 )
@@ -19,15 +20,15 @@ func (t *defaultModelBuildTask) buildListenerRules(ctx context.Context, lsARN co
 for _, path := range rule.HTTP.Paths {
 enhancedBackend, err := t.enhancedBackendBuilder.Build(ctx, ing, path.Backend)
 if err != nil {
-return err
+return errors.Wrapf(err, "ingress: %v", k8s.NamespacedName(ing))
 }
 conditions, err := t.buildRuleConditions(ctx, rule, path, enhancedBackend)
 if err != nil {
-return err
+return errors.Wrapf(err, "ingress: %v", k8s.NamespacedName(ing))
 }
 actions, err := t.buildActions(ctx, protocol, ing, enhancedBackend)
 if err != nil {
-return err
+return errors.Wrapf(err, "ingress: %v", k8s.NamespacedName(ing))
 }
 rules = append(rules, Rule{
 Conditions: conditions,

pkg/ingress/model_builder.go

@@ -160,7 +160,7 @@ func (t *defaultModelBuildTask) run(ctx context.Context) error {
 for _, ing := range t.ingGroup.Members {
 listenPortConfigByPortForIngress, err := t.computeIngressListenPortConfigByPort(ctx, ing)
 if err != nil {
-return err
+return errors.Wrapf(err, "ingress: %v", k8s.NamespacedName(ing))
 }
 ingKey := k8s.NamespacedName(ing)
 for port, cfg := range listenPortConfigByPortForIngress {

pkg/k8s/events.go

@@ -1,5 +1,27 @@
 package k8s
 
 const (
-IngressEventReasonFailedToLoadGroupID = "FailedToLoadGroupID"
+// Ingress events
+IngressEventReasonFailedLoadGroupID = "FailedLoadGroupID"
+IngressEventReasonFailedAddFinalizer = "FailedAddFinalizer"
+IngressEventReasonFailedRemoveFinalizer = "FailedRemoveFinalizer"
+IngressEventReasonFailedUpdateStatus = "FailedUpdateStatus"
+IngressEventReasonFailedBuildModel = "FailedBuildModel"
+IngressEventReasonFailedDeployModel = "FailedDeployModel"
+IngressEventReasonSuccessfullyReconciled = "SuccessfullyReconciled"
+
+// Service events
+ServiceEventReasonFailedAddFinalizer = "FailedAddFinalizer"
+ServiceEventReasonFailedRemoveFinalizer = "FailedRemoveFinalizer"
+ServiceEventReasonFailedUpdateStatus = "FailedUpdateStatus"
+ServiceEventReasonFailedBuildModel = "FailedBuildModel"
+ServiceEventReasonFailedDeployModel = "FailedDeployModel"
+ServiceEventReasonSuccessfullyReconciled = "SuccessfullyReconciled"
+
+// TargetGroupBinding events
+TargetGroupBindingEventReasonFailedAddFinalizer = "FailedAddFinalizer"
+TargetGroupBindingEventReasonFailedRemoveFinalizer = "FailedRemoveFinalizer"
+TargetGroupBindingEventReasonFailedUpdateStatus = "FailedUpdateStatus"
+TargetGroupBindingEventReasonFailedCleanup = "FailedCleanup"
+TargetGroupBindingEventReasonSuccessfullyReconciled = "SuccessfullyReconciled"
 )