Enable passing config into controller (#264)

* Enable passing input into controller * Enable passing input into controller * Improve input pass to controller * Change flag to env var for input * Remove checking if running inside cluster code * Update deploy docs * Update default setting * Update default setting * Update default setting * Add unit test for env var init

Author: zmingxi

docs/deploy.md

@@ -14,7 +14,7 @@ Run through them again for a second cluster to use with the extended example sho
  ```bash
  eksctl create cluster --name $CLUSTER_NAME --region $AWS_REGION
  ```
-1. First, configure security group to receive traffic from the VPC Lattice fleet. You must set up security groups so that they allow all Pods communicating with VPC Lattice to allow traffic on all ports from the `169.254.171.0/24` address range. 
+1. First, configure security group to receive traffic from the VPC Lattice fleet. You must set up security groups so that they allow all Pods communicating with VPC Lattice to allow traffic on all ports from the `169.254.171.0/24` address range.
  ```bash
  PREFIX_LIST_ID=$(aws ec2 describe-managed-prefix-lists --query "PrefixLists[?PrefixListName=="\'com.amazonaws.$AWS_REGION.vpc-lattice\'"].PrefixListId" | jq -r '.[]')
  MANAGED_PREFIX=$(aws ec2 get-managed-prefix-list-entries --prefix-list-id $PREFIX_LIST_ID --output json | jq -r '.Entries[0].Cidr')
@@ -79,7 +79,12 @@ Run through them again for a second cluster to use with the extended example sho
  helm install gateway-api-controller \
  oci://public.ecr.aws/aws-application-networking-k8s/aws-gateway-controller-chart\
  --version=v0.0.12 \
- --set=aws.region=$AWS_REGION --set=serviceAccount.create=false --namespace aws-application-networking-system
+ --set=serviceAccount.create=false --namespace aws-application-networking-system \
+ # Region, clusterVpcId, awsAccountId are required for case where IMDS is NOT AVAILABLE, e.g Fargate
+ --set=awsRegion= \
+ --set=clusterVpcId= \
+ --set=awsAccountId= \
+
  ```
 1. Create the `amazon-vpc-lattice` GatewayClass:
  ```bash

helm/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: env-config
+data:
+ awsRegion: {{ .Values.awsRegion }}
+ awsAccountId: {{ .Values.awsAccountId }}
+ clusterVpcId: {{ .Values.clusterVpcId }}

helm/templates/deployment.yaml

@@ -66,6 +66,23 @@ spec:
  periodSeconds: 10
  securityContext:
  allowPrivilegeEscalation: false
+ env:
+ - name: REGION
+ valueFrom:
+ configMapKeyRef:
+ name: env-config
+ key: awsRegion
+ - name: AWS_ACCOUNT_ID
+ valueFrom:
+ configMapKeyRef:
+ name: env-config
+ key: awsAccountId
+ - name: CLUSTER_VPC_ID
+ valueFrom:
+ configMapKeyRef:
+ name: env-config
+ key: clusterVpcId
+
  terminationGracePeriodSeconds: 10
  nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }}
  {{ if .Values.deployment.tolerations -}}

helm/values.yaml

@@ -70,3 +70,7 @@ serviceAccount:
  name: gateway-api-controller
  annotations: {}
  # eks.amazonaws.com/role-arn: arn:aws:iam::AWS_ACCOUNT_ID:role/IAM_ROLE_NAME
+
+awsRegion:
+awsAccountId:
+clusterVpcId:

pkg/config/controller_config.go

@@ -12,20 +12,28 @@ import (
 const (
 LatticeGatewayControllerName = "application-networking.k8s.aws/gateway-api-controller"
 defaultLogLevel = "Info"
-NoDefaultServiceNetwork = ""
-NO_DEFAULT_SERVICE_NETWORK = "NO_DEFAULT_SERVICE_NETWORK"
+UnknownInput = ""
 )
 
-// TODO endpoint, region
-var VpcID = "vpc-xxxx"
-var AccountID = "yyyyyy"
-var Region = "us-west-2"
+const (
+NO_DEFAULT_SERVICE_NETWORK = "NO_DEFAULT_SERVICE_NETWORK"
+REGION = "REGION"
+CLUSTER_VPC_ID = "CLUSTER_VPC_ID"
+CLUSTER_LOCAL_GATEWAY = "CLUSTER_LOCAL_GATEWAY"
+AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID"
+TARGET_GROUP_NAME_LEN_MODE = "TARGET_GROUP_NAME_LEN_MODE"
+GATEWAY_API_CONTROLLER_LOGLEVEL = "GATEWAY_API_CONTROLLER_LOGLEVEL"
+)
+
+var VpcID = UnknownInput
+var AccountID = UnknownInput
+var Region = UnknownInput
 var logLevel = defaultLogLevel
-var DefaultServiceNetwork = NoDefaultServiceNetwork
+var DefaultServiceNetwork = UnknownInput
 var UseLongTGName = false
 
 func GetLogLevel() string {
-logLevel = os.Getenv("GATEWAY_API_CONTROLLER_LOGLEVEL")
+logLevel = os.Getenv(GATEWAY_API_CONTROLLER_LOGLEVEL)
 switch strings.ToLower(logLevel) {
 case "debug":
 return "10"
@@ -36,86 +44,74 @@ func GetLogLevel() string {
 }
 
 func GetClusterLocalGateway() (string, error) {
-if DefaultServiceNetwork == NoDefaultServiceNetwork {
-return NoDefaultServiceNetwork, errors.New(NO_DEFAULT_SERVICE_NETWORK)
+if DefaultServiceNetwork == UnknownInput {
+return UnknownInput, errors.New(NO_DEFAULT_SERVICE_NETWORK)
 }
 
 return DefaultServiceNetwork, nil
 }
 
 func ConfigInit() {
-// discover VPC using environment first
-VpcID = os.Getenv("CLUSTER_VPC_ID")
-glog.V(2).Infoln("CLUSTER_VPC_ID: ", os.Getenv("CLUSTER_VPC_ID"))
-
-// discover Account
-AccountID = os.Getenv("AWS_ACCOUNT_ID")
-if AccountID == "" {
-AccountID = os.Getenv("AWS_ACCOUNT") // Fallback to AWS_ACCOUNT for compatibility
-}
-glog.V(2).Infoln("AWS_ACCOUNT_ID:", AccountID)
-
-// discover Region
-Region = os.Getenv("REGION")
-glog.V(2).Infoln("REGION:", os.Getenv("REGION"))
-
-logLevel = os.Getenv("GATEWAY_API_CONTROLLER_LOGLEVEL")
-glog.V(2).Infoln("Logging Level:", os.Getenv("GATEWAY_API_CONTROLLER_LOGLEVEL"))
-
-DefaultServiceNetwork = os.Getenv("CLUSTER_LOCAL_GATEWAY")
-
-if DefaultServiceNetwork == NoDefaultServiceNetwork {
-glog.V(2).Infoln("No CLUSTER_LOCAL_GATEWAY")
-} else {
-
-glog.V(2).Infoln("CLUSTER_LOCAL_GATEWAY", DefaultServiceNetwork)
-}
-
-tgNameLengthMode := os.Getenv("TARGET_GROUP_NAME_LEN_MODE")
-
-glog.V(2).Infoln("TARGET_GROUP_NAME_LEN_MODE", tgNameLengthMode)
-
-if tgNameLengthMode == "long" {
-UseLongTGName = true
-} else {
-UseLongTGName = false
-}
 
 sess, _ := session.NewSession()
 metadata := NewEC2Metadata(sess)
-
 var err error
-if ifRunningInCluster() {
+
+// CLUSTER_VPC_ID
+VpcID = os.Getenv(CLUSTER_VPC_ID)
+if VpcID != UnknownInput {
+glog.V(2).Infoln("CLUSTER_VPC_ID passed as input:", VpcID)
+} else {
 VpcID, err = metadata.VpcID()
+glog.V(2).Infoln("CLUSTER_VPC_ID from IMDS config discovery :", VpcID)
 if err != nil {
-return
+glog.V(2).Infoln("IMDS config discovery for CLUSTER_VPC_ID is NOT AVAILABLE :", err)
 }
+}
+
+// REGION
+Region = os.Getenv(REGION)
+if Region != UnknownInput {
+glog.V(2).Infoln("REGION passed as input:", Region)
+} else {
 Region, err = metadata.Region()
+glog.V(2).Infoln("REGION from IMDS config discovery :", Region)
 if err != nil {
-return
+glog.V(2).Infoln("IMDS config discovery for REGION is NOT AVAILABLE :", err)
 }
+}
+
+// AWS_ACCOUNT_ID
+AccountID = os.Getenv(AWS_ACCOUNT_ID)
+if AccountID != UnknownInput {
+glog.V(2).Infoln("AWS_ACCOUNT_ID passed as input:", AccountID)
+} else {
 AccountID, err = metadata.AccountId()
+glog.V(2).Infoln("AWS_ACCOUNT_ID from IMDS config discovery :", AccountID)
 if err != nil {
-return
+glog.V(2).Infoln("IMDS config discovery for AWS_ACCOUNT_ID is NOT AVAILABLE :", err)
 }
-glog.V(2).Infoln("INSIDE CLUSTER CLUSTER_VPC_ID: ", VpcID)
-glog.V(2).Infoln("INSIDE CLUSTER REGION: ", Region)
-glog.V(2).Infoln("INSIDE CLUSTER ACCOUNT_ID: ", AccountID)
 }
-}
 
-func ifRunningInCluster() bool {
-_, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount")
-if err == nil {
-glog.V(2).Infoln("Controller is running inside cluster")
-return true
-}
+// GATEWAY_API_CONTROLLER_LOGLEVEL
+logLevel = os.Getenv(GATEWAY_API_CONTROLLER_LOGLEVEL)
+glog.V(2).Infoln("Logging Level:", os.Getenv(GATEWAY_API_CONTROLLER_LOGLEVEL))
 
-if os.IsNotExist(err) {
-glog.V(2).Infoln("Controller is NOT running inside cluster")
-return false
+// CLUSTER_LOCAL_GATEWAY
+DefaultServiceNetwork = os.Getenv(CLUSTER_LOCAL_GATEWAY)
+if DefaultServiceNetwork == UnknownInput {
+glog.V(2).Infoln("No CLUSTER_LOCAL_GATEWAY")
+} else {
+glog.V(2).Infoln("CLUSTER_LOCAL_GATEWAY", DefaultServiceNetwork)
 }
 
-glog.V(2).Infoln("Controller is NOT running inside cluster")
-return false
+// TARGET_GROUP_NAME_LEN_MODE
+tgNameLengthMode := os.Getenv(TARGET_GROUP_NAME_LEN_MODE)
+glog.V(2).Infoln("TARGET_GROUP_NAME_LEN_MODE", tgNameLengthMode)
+
+if tgNameLengthMode == "long" {
+UseLongTGName = true
+} else {
+UseLongTGName = false
+}
 }

pkg/config/controller_config_test.go

@@ -0,0 +1,61 @@
+package config
+
+import (
+"github.com/stretchr/testify/assert"
+"os"
+"testing"
+)
+
+func Test_config_init_with_partial_env_var(t *testing.T) {
+// Test variable
+testRegion := "us-west-2"
+testClusterVpcId := "vpc-123456"
+testClusterLocalGateway := "default"
+
+os.Setenv(REGION, testRegion)
+os.Setenv(CLUSTER_VPC_ID, testClusterVpcId)
+os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway)
+os.Unsetenv(AWS_ACCOUNT_ID)
+os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE)
+ConfigInit()
+assert.Equal(t, Region, testRegion)
+assert.Equal(t, VpcID, testClusterVpcId)
+assert.Equal(t, AccountID, UnknownInput)
+assert.Equal(t, DefaultServiceNetwork, testClusterLocalGateway)
+assert.Equal(t, UseLongTGName, false)
+}
+
+func Test_config_init_no_env_var(t *testing.T) {
+os.Unsetenv(REGION)
+os.Unsetenv(CLUSTER_VPC_ID)
+os.Unsetenv(CLUSTER_LOCAL_GATEWAY)
+os.Unsetenv(AWS_ACCOUNT_ID)
+os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE)
+ConfigInit()
+assert.Equal(t, Region, UnknownInput)
+assert.Equal(t, VpcID, UnknownInput)
+assert.Equal(t, AccountID, UnknownInput)
+assert.Equal(t, DefaultServiceNetwork, UnknownInput)
+assert.Equal(t, UseLongTGName, false)
+}
+
+func Test_config_init_with_all_env_var(t *testing.T) {
+// Test variable
+testRegion := "us-west-2"
+testClusterVpcId := "vpc-123456"
+testClusterLocalGateway := "default"
+testTargetGroupNameLenMode := "long"
+testAwsAccountId := "12345678"
+
+os.Setenv(REGION, testRegion)
+os.Setenv(CLUSTER_VPC_ID, testClusterVpcId)
+os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway)
+os.Setenv(AWS_ACCOUNT_ID, testAwsAccountId)
+os.Setenv(TARGET_GROUP_NAME_LEN_MODE, testTargetGroupNameLenMode)
+ConfigInit()
+assert.Equal(t, Region, testRegion)
+assert.Equal(t, VpcID, testClusterVpcId)
+assert.Equal(t, AccountID, testAwsAccountId)
+assert.Equal(t, DefaultServiceNetwork, testClusterLocalGateway)
+assert.Equal(t, UseLongTGName, true)
+}

pkg/config/ec2_metadata.go

@@ -34,7 +34,6 @@ func (c *defaultEC2Metadata) VpcID() (string, error) {
 if err != nil {
 return "", err
 }
-fmt.Println("Get VPC ID from ec2 metadata: ", vpcID)
 return vpcID, nil
 }
 
@@ -43,7 +42,6 @@ func (c *defaultEC2Metadata) Region() (string, error) {
 if err != nil {
 return "", err
 }
-fmt.Println("Get region from ec2 metadata: ", region)
 return region, nil
 }