monitor secret resources ony if necessary (kubernetes-sigs#2550)

* monitor secret resources ony if necessary * removes the default watch for all secret resources in the cluster * create watch for specific secret resource if required for OIDC feature * do not configure rbac permissions for secrets by default * provide helm chart configuration options for specific secret resources * remove option to configure specific secret names * use channel for secrets event

Author: kishorj

config/rbac/role.yaml

@@ -52,14 +52,6 @@ rules:
  verbs:
  - patch
  - update
-- apiGroups:
- - ""
- resources:
- - secrets
- verbs:
- - get
- - list
- - watch
 - apiGroups:
  - ""
  resources:

controllers/ingress/eventhandlers/secret_events.go

@@ -63,7 +63,8 @@ func (h *enqueueRequestsForSecretEvent) Delete(e event.DeleteEvent, _ workqueue.
 }
 
 func (h *enqueueRequestsForSecretEvent) Generic(e event.GenericEvent, _ workqueue.RateLimitingInterface) {
-// we don't have any generic event for secrets.
+secretObj := e.Object.(*corev1.Secret)
+h.enqueueImpactedObjects(secretObj)
 }
 
 func (h *enqueueRequestsForSecretEvent) enqueueImpactedObjects(secret *corev1.Secret) {

controllers/ingress/group_controller.go

@@ -93,6 +93,7 @@ type groupReconciler struct {
 stackMarshaller deploy.StackMarshaller
 stackDeployer deploy.StackDeployer
 backendSGProvider networkingpkg.BackendSGProvider
+secretsManager k8s.SecretsManager
 
 groupLoader ingress.GroupLoader
 groupFinalizerManager ingress.FinalizerManager
@@ -160,7 +161,7 @@ func (r *groupReconciler) reconcile(ctx context.Context, req ctrl.Request) error
 }
 
 func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) {
-stack, lb, err := r.modelBuilder.Build(ctx, ingGroup)
+stack, lb, secrets, err := r.modelBuilder.Build(ctx, ingGroup)
 if err != nil {
 r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
 return nil, nil, err
@@ -177,6 +178,7 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
 return nil, nil, err
 }
 r.logger.Info("successfully deployed model", "ingressGroup", ingGroup.ID)
+r.secretsManager.MonitorSecrets(ingGroup.ID.String(), secrets)
 return stack, lb, err
 }
 
@@ -229,7 +231,7 @@ func (r *groupReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager
 if err := r.setupIndexes(ctx, mgr.GetFieldIndexer(), ingressClassResourceAvailable); err != nil {
 return err
 }
-if err := r.setupWatches(ctx, c, ingressClassResourceAvailable); err != nil {
+if err := r.setupWatches(ctx, c, ingressClassResourceAvailable, clientSet); err != nil {
 return err
 }
 return nil
@@ -276,9 +278,10 @@ func (r *groupReconciler) setupIndexes(ctx context.Context, fieldIndexer client.
 return nil
 }
 
-func (r *groupReconciler) setupWatches(_ context.Context, c controller.Controller, ingressClassResourceAvailable bool) error {
+func (r *groupReconciler) setupWatches(_ context.Context, c controller.Controller, ingressClassResourceAvailable bool, clientSet *kubernetes.Clientset) error {
 ingEventChan := make(chan event.GenericEvent)
 svcEventChan := make(chan event.GenericEvent)
+secretEventsChan := make(chan event.GenericEvent)
 ingEventHandler := eventhandlers.NewEnqueueRequestsForIngressEvent(r.groupLoader, r.eventRecorder,
 r.logger.WithName("eventHandlers").WithName("ingress"))
 svcEventHandler := eventhandlers.NewEnqueueRequestsForServiceEvent(ingEventChan, r.k8sClient, r.eventRecorder,
@@ -297,10 +300,9 @@ func (r *groupReconciler) setupWatches(_ context.Context, c controller.Controlle
 if err := c.Watch(&source.Kind{Type: &corev1.Service{}}, svcEventHandler); err != nil {
 return err
 }
-if err := c.Watch(&source.Kind{Type: &corev1.Secret{}}, secretEventHandler); err != nil {
+if err := c.Watch(&source.Channel{Source: secretEventsChan}, secretEventHandler); err != nil {
 return err
 }
-
 if ingressClassResourceAvailable {
 ingClassEventChan := make(chan event.GenericEvent)
 ingClassParamsEventHandler := eventhandlers.NewEnqueueRequestsForIngressClassParamsEvent(ingClassEventChan, r.k8sClient, r.eventRecorder,
@@ -317,6 +319,7 @@ func (r *groupReconciler) setupWatches(_ context.Context, c controller.Controlle
 return err
 }
 }
+r.secretsManager = k8s.NewSecretsManager(clientSet, secretEventsChan, ctrl.Log.WithName("secrets-manager"))
 return nil
 }
 

helm/aws-load-balancer-controller/README.md

@@ -234,3 +234,4 @@ The default values set by the application itself can be confirmed [here](https:/
 | `serviceMonitor.enabled` | Specifies whether a service monitor should be created, requires the ServiceMonitor CRD to be installed | `false` |
 | `serviceMonitor.additionalLabels` | Labels to add to the service account | `{}` |
 | `serviceMonitor.interval` | Prometheus scrape interval | `1m` |
+| `clusterSecretsPermissions.allowAllSecrets` | If `true`, controller has access to all secrets in the cluster. | `false` |

helm/aws-load-balancer-controller/templates/rbac.yaml

@@ -57,8 +57,13 @@ rules:
  resources: [services, ingresses]
  verbs: [get, list, patch, update, watch]
 - apiGroups: [""]
- resources: [nodes, secrets, namespaces, endpoints]
+ resources: [nodes, namespaces, endpoints]
  verbs: [get, list, watch]
+{{- if .Values.clusterSecretsPermissions.allowAllSecrets }}
+- apiGroups: [""]
+ resources: [secrets]
+ verbs: [get, list, watch]
+{{- end }}
 - apiGroups: ["elbv2.k8s.aws", "", "extensions", "networking.k8s.io"]
  resources: [targetgroupbindings/status, pods/status, services/status, ingresses/status]
  verbs: [update, patch]

helm/aws-load-balancer-controller/values.yaml

@@ -266,3 +266,12 @@ serviceMonitor:
  additionalLabels: {}
  # Prometheus scrape interval
  interval: 1m
+
+# clusterSecretsPermissions lets you configure RBAC permissions for secret resources
+# Access to secrets resource is required only if you use the OIDC feature, and instead of
+# enabling access to all secrets, we recommend configuring namespaced role/rolebinding.
+# This option is for backwards compatibility only, and will potentially be deprecated in future.
+clusterSecretsPermissions:
+ # allowAllSecrets allows the controller to access all secrets in the cluster.
+ # This is to get backwards compatible behavior, but *NOT* recommended for security reasons
+ allowAllSecrets: false

pkg/config/runtime_config.go

@@ -1,14 +1,16 @@
 package config
 
 import (
+"time"
+
 "github.com/spf13/pflag"
 corev1 "k8s.io/api/core/v1"
 "k8s.io/apimachinery/pkg/runtime"
 "k8s.io/client-go/rest"
 "k8s.io/client-go/tools/clientcmd"
 "k8s.io/client-go/tools/leaderelection/resourcelock"
 ctrl "sigs.k8s.io/controller-runtime"
-"time"
+"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -121,6 +123,7 @@ func BuildRuntimeOptions(rtCfg RuntimeConfig, scheme *runtime.Scheme) ctrl.Optio
 LeaderElectionNamespace: rtCfg.LeaderElectionNamespace,
 Namespace: rtCfg.WatchNamespace,
 SyncPeriod: &rtCfg.SyncPeriod,
+ClientDisableCacheFor: []client.Object{&corev1.Secret{}},
 }
 }
 

pkg/ingress/model_build_actions.go

@@ -172,7 +172,6 @@ func (t *defaultModelBuildTask) buildAuthenticateOIDCAction(ctx context.Context,
 if err := t.k8sClient.Get(ctx, secretKey, secret); err != nil {
 return elbv2model.Action{}, err
 }
-
 rawClientID, ok := secret.Data["clientID"]
 // AWSALBIngressController looks for clientId, we should be backwards-compatible here.
 if !ok {
@@ -186,6 +185,7 @@ func (t *defaultModelBuildTask) buildAuthenticateOIDCAction(ctx context.Context,
 return elbv2model.Action{}, errors.Errorf("missing clientSecret, secret: %v", secretKey)
 }
 
+t.secretKeys = append(t.secretKeys, secretKey)
 clientID := strings.TrimRightFunc(string(rawClientID), unicode.IsSpace)
 clientSecret := string(rawClientSecret)
 return elbv2model.Action{

pkg/ingress/model_builder.go

@@ -30,7 +30,7 @@ const (
 // ModelBuilder is responsible for build mode stack for a IngressGroup.
 type ModelBuilder interface {
 // build mode stack for a IngressGroup.
-Build(ctx context.Context, ingGroup Group) (core.Stack, *elbv2model.LoadBalancer, error)
+Build(ctx context.Context, ingGroup Group) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, error)
 }
 
 // NewDefaultModelBuilder constructs new defaultModelBuilder.
@@ -97,7 +97,7 @@ type defaultModelBuilder struct {
 }
 
 // build mode stack for a IngressGroup.
-func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.Stack, *elbv2model.LoadBalancer, error) {
+func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, error) {
 stack := core.NewDefaultStack(core.StackID(ingGroup.ID))
 task := &defaultModelBuildTask{
 k8sClient: b.k8sClient,
@@ -143,9 +143,9 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.S
 backendServices: make(map[types.NamespacedName]*corev1.Service),
 }
 if err := task.run(ctx); err != nil {
-return nil, nil, err
+return nil, nil, nil, err
 }
-return task.stack, task.loadBalancer, nil
+return task.stack, task.loadBalancer, task.secretKeys, nil
 }
 
 // the default model build task
@@ -193,6 +193,7 @@ type defaultModelBuildTask struct {
 loadBalancer *elbv2model.LoadBalancer
 tgByResID map[string]*elbv2model.TargetGroup
 backendServices map[types.NamespacedName]*corev1.Service
+secretKeys []types.NamespacedName
 }
 
 func (t *defaultModelBuildTask) run(ctx context.Context) error {

pkg/ingress/model_builder_test.go

@@ -3900,7 +3900,7 @@ func Test_defaultModelBuilder_Build(t *testing.T) {
 defaultSSLPolicy: "ELBSecurityPolicy-2016-08",
 }
 
-gotStack, _, err := b.Build(context.Background(), tt.args.ingGroup)
+gotStack, _, _, err := b.Build(context.Background(), tt.args.ingGroup)
 if tt.wantErr != nil {
 assert.EqualError(t, err, tt.wantErr.Error())
 } else {