Support disabling api gateway in cluster config (#1259)

Author: deliahu

cli/cmd/cluster.go

@@ -154,20 +154,26 @@ var _upCmd = &cobra.Command{
 exit.Error(err)
 }
 
-err = createOrReplaceAPIGateway(awsClient, clusterConfig.ClusterName, clusterConfig.Tags)
-if err != nil {
-exit.Error(err)
+if clusterConfig.APIGatewaySetting == clusterconfig.EnabledAPIGatewaySetting {
+err = createOrReplaceAPIGateway(awsClient, clusterConfig.ClusterName, clusterConfig.Tags)
+if err != nil {
+exit.Error(err)
+}
 }
 
 out, exitCode, err := runManagerUpdateCommand("/root/install.sh", clusterConfig, awsCreds, _flagClusterEnv)
 if err != nil {
-awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
-awsClient.DeleteVPCLinkByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+if clusterConfig.APIGatewaySetting == clusterconfig.EnabledAPIGatewaySetting {
+awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+awsClient.DeleteVPCLinkByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+}
 exit.Error(err)
 }
 if exitCode == nil || *exitCode != 0 {
-awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
-awsClient.DeleteVPCLinkByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+if clusterConfig.APIGatewaySetting == clusterconfig.EnabledAPIGatewaySetting {
+awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+awsClient.DeleteVPCLinkByTag(clusterconfig.ClusterNameTag, clusterConfig.ClusterName) // best effort deletion
+}
 helpStr := "\nDebugging tips (may or may not apply to this error):"
 helpStr += fmt.Sprintf("\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", *clusterConfig.Region)
 helpStr += fmt.Sprintf("\n* if your cluster started spinning up, please ensure that your CloudFormation stacks for this cluster have been fully deleted before trying to spin up this cluster again (you can delete your CloudFormation stacks from the AWS console: %s)", getCloudFormationURL(clusterConfig.ClusterName, *clusterConfig.Region))
@@ -322,7 +328,7 @@ var _downCmd = &cobra.Command{
 }
 
 fmt.Print("￮ deleting api gateway ")
-_, errAPIGateway := awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, *accessConfig.ClusterName)
+deletedAPIGateway, errAPIGateway := awsClient.DeleteAPIGatewayByTag(clusterconfig.ClusterNameTag, *accessConfig.ClusterName)
 _, errVPCLink := awsClient.DeleteVPCLinkByTag(clusterconfig.ClusterNameTag, *accessConfig.ClusterName)
 if errAPIGateway != nil {
 fmt.Printf("\n\nunable to delete cortex's api gateway (see error below); if it still exists after the cluster has been deleted, please delete it via the api gateway console: https://%s.console.aws.amazon.com/apigateway/main/apis\n", *accessConfig.Region)
@@ -333,7 +339,11 @@ var _downCmd = &cobra.Command{
 errors.PrintError(errVPCLink)
 }
 if errAPIGateway == nil && errVPCLink == nil {
-fmt.Println("✓")
+if deletedAPIGateway != nil {
+fmt.Println("✓")
+} else {
+fmt.Println("(n/a)")
+}
 } else {
 fmt.Println()
 }

cli/cmd/lib_cluster_config.go

@@ -315,6 +315,11 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 }
 userClusterConfig.OperatorLoadBalancerScheme = cachedClusterConfig.OperatorLoadBalancerScheme
 
+if userClusterConfig.APIGatewaySetting != cachedClusterConfig.APIGatewaySetting {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.APIGatewaySettingKey, cachedClusterConfig.APIGatewaySetting)
+}
+userClusterConfig.APIGatewaySetting = cachedClusterConfig.APIGatewaySetting
+
 if userClusterConfig.Spot != nil && *userClusterConfig.Spot != *cachedClusterConfig.Spot {
 return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.SpotKey, *cachedClusterConfig.Spot)
 }
@@ -459,6 +464,10 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 }
 fmt.Printf("cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\n\n", clusterConfig.Bucket, clusterConfig.LogGroup, privateSubnetMsg)
 
+if clusterConfig.APIGatewaySetting == clusterconfig.DisabledAPIGatewaySetting {
+fmt.Print("warning: you've disabled API Gateway cluster-wide, so APIs will not be able to create API Gateway endpoints (they will still be reachable via the API load balancer; see https://docs.cortex.dev/deployments/networking for more information)\n\n")
+}
+
 if clusterConfig.OperatorLoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme {
 fmt.Print("warning: you've configured the operator load balancer to be internal; you must configure VPC Peering to connect your CLI to your cluster operator (see https://docs.cortex.dev/guides/vpc-peering)\n\n")
 }
@@ -535,6 +544,9 @@ func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds A
 if clusterConfig.OperatorLoadBalancerScheme != defaultConfig.OperatorLoadBalancerScheme {
 items.Add(clusterconfig.OperatorLoadBalancerSchemeUserKey, clusterConfig.OperatorLoadBalancerScheme)
 }
+if clusterConfig.APIGatewaySetting != defaultConfig.APIGatewaySetting {
+items.Add(clusterconfig.APIGatewaySettingUserKey, clusterConfig.APIGatewaySetting)
+}
 
 if clusterConfig.Spot != nil && *clusterConfig.Spot != *defaultConfig.Spot {
 items.Add(clusterconfig.SpotUserKey, s.YesNo(clusterConfig.Spot != nil && *clusterConfig.Spot))

docs/cluster-management/config.md

@@ -66,6 +66,11 @@ api_load_balancer_scheme: internet-facing # must be "internet-facing" or "inter
 # see https://docs.cortex.dev/v/master/miscellaneous/security#private-cluster for more information
 operator_load_balancer_scheme: internet-facing # must be "internet-facing" or "internal"
 
+# whether to disable API gateway cluster-wide
+# if set to "enabled" (the default), each API can specify whether to use API Gateway
+# if set to "disabled", no APIs will be allowed to use API Gateway
+api_gateway: enabled # must be "enabled" or "disabled"
+
 # CloudWatch log group for cortex (default: <cluster_name>)
 log_group: cortex
 

docs/deployments/networking.md

@@ -4,7 +4,7 @@ _WARNING: you are on the master branch, please refer to the docs on the branch t
 
 ![api architecture diagram](https://user-images.githubusercontent.com/808475/84695323-8507dd00-aeff-11ea-8b32-5a55cef76c79.png)
 
-APIs are deployed with a public API Gateway by default (the API Gateway forwards requests to the API load balancer). Each API can be independently configured to not create the API Gateway endpoint by setting `api_gateway: none` in the `networking` field of the [api configuration](api-configuration.md). If the API Gateway endpoint is not created, your API can still be accessed via the API load balancer; `cortex get API_NAME` will show the load balancer endpoint if API Gateway is disabled. API Gateway is enabled by default, and is generally recommended unless it doesn't support your use case due to limitations such as the 29 second request timeout, or if you are keeping your APIs private to your VPC. See below for common configurations.
+APIs are deployed with a public API Gateway by default (the API Gateway forwards requests to the API load balancer). Each API can be independently configured to not create the API Gateway endpoint by setting `api_gateway: none` in the `networking` field of the [api configuration](api-configuration.md). If the API Gateway endpoint is not created, your API can still be accessed via the API load balancer; `cortex get API_NAME` will show the load balancer endpoint if API Gateway is disabled. API Gateway is enabled by default, and is generally recommended unless it doesn't support your use case due to limitations such as the 29 second request timeout, or if you are keeping your APIs private to your VPC. See below for common configurations. To disable API Gateway cluster-wide (thereby enforcing that all APIs cannot create API Gateway endpoints), set `api_gateway: disabled` in your [cluster configuration](../cluster-management/config.md) file (before creating your cluster).
 
 By default, the API load balancer is public. You can configure your API load balancer to be private by setting `api_load_balancer_scheme: internal` in your [cluster configuration](../cluster-management/config.md) file (before creating your cluster). This will force external traffic to go through your API Gateway endpoint, or if you disabled API Gateway for your API, it will make your API only accessible through VPC Peering. Note that if API Gateway is used, endpoints will be public regardless of `api_load_balancer_scheme`. See below for common configurations.
 

manager/install.sh

@@ -167,7 +167,7 @@ function main() {
  ensure_eks
 
  # create VPC Link for API Gateway
- if [ "$arg1" != "--update" ] && [ "$CORTEX_API_LOAD_BALANCER_SCHEME" == "internal" ]; then
+ if [ "$arg1" != "--update" ] && [ "$CORTEX_API_LOAD_BALANCER_SCHEME" == "internal" ] && [ "$CORTEX_API_GATEWAY" == "enabled" ]; then
  vpc_id=$(aws ec2 describe-vpcs --region $CORTEX_REGION --filters Name=tag:eksctl.cluster.k8s.io/v1alpha1/cluster-name,Values=$CORTEX_CLUSTER_NAME | jq .Vpcs[0].VpcId | tr -d '"')
  if [ "$vpc_id" = "" ] || [ "$vpc_id" = "null" ]; then
  echo "unable to find cortex vpc"
@@ -247,7 +247,7 @@ function main() {
  fi
 
  # add VPC Link integration to API Gateway
- if [ "$arg1" != "--update" ] && [ "$CORTEX_API_LOAD_BALANCER_SCHEME" == "internal" ]; then
+ if [ "$arg1" != "--update" ] && [ "$CORTEX_API_LOAD_BALANCER_SCHEME" == "internal" ] && [ "$CORTEX_API_GATEWAY" == "enabled" ]; then
  echo -n "￮ creating api gateway vpc link integration "
  api_id=$(python get_api_gateway_id.py)
  python create_gateway_integration.py $api_id $vpc_link_id

pkg/lib/aws/apigateway.go

@@ -111,7 +111,7 @@ func (c *Client) GetAPIGatewayByTag(tagName string, tagValue string) (*apigatewa
 return nil, nil
 }
 
-// DeleteVPCLinkByTag Deletes a VPC Link by tag (returns the deleted VPC Link, or nil if it was not found )
+// DeleteVPCLinkByTag Deletes a VPC Link by tag (returns the deleted VPC Link, or nil if it was not found)
 func (c *Client) DeleteVPCLinkByTag(tagName string, tagValue string) (*apigatewayv2.VpcLink, error) {
 vpcLink, err := c.GetVPCLinkByTag(tagName, tagValue)
 if err != nil {
@@ -130,7 +130,7 @@ func (c *Client) DeleteVPCLinkByTag(tagName string, tagValue string) (*apigatewa
 return vpcLink, nil
 }
 
-// DeleteAPIGatewayByTag Deletes an API Gateway by tag (returns whether or not the API Gateway existed)
+// DeleteAPIGatewayByTag Deletes an API Gateway by tag (returns the deleted API Gateway, or nil if it was not found)
 func (c *Client) DeleteAPIGatewayByTag(tagName string, tagValue string) (*apigatewayv2.Api, error) {
 apiGateway, err := c.GetAPIGatewayByTag(tagName, tagValue)
 if err != nil {

pkg/operator/config/config.go

@@ -86,30 +86,32 @@ func Init() error {
 fmt.Println(errors.Message(err))
 }
 
-apiGateway, err := AWS.GetAPIGatewayByTag(clusterconfig.ClusterNameTag, Cluster.ClusterName)
-if err != nil {
-return err
-} else if apiGateway == nil {
-return ErrorNoAPIGateway()
-}
-Cluster.APIGateway = *apiGateway
-
-if Cluster.APILoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme {
-vpcLink, err := AWS.GetVPCLinkByTag(clusterconfig.ClusterNameTag, Cluster.ClusterName)
+if Cluster.APIGatewaySetting == clusterconfig.EnabledAPIGatewaySetting {
+apiGateway, err := AWS.GetAPIGatewayByTag(clusterconfig.ClusterNameTag, Cluster.ClusterName)
 if err != nil {
 return err
-} else if vpcLink == nil {
-return ErrorNoVPCLink()
+} else if apiGateway == nil {
+return ErrorNoAPIGateway()
 }
-Cluster.VPCLink = vpcLink
-
-integration, err := AWS.GetVPCLinkIntegration(*Cluster.APIGateway.ApiId, *Cluster.VPCLink.VpcLinkId)
-if err != nil {
-return err
-} else if integration == nil {
-return ErrorNoVPCLinkIntegration()
+Cluster.APIGateway = apiGateway
+
+if Cluster.APILoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme {
+vpcLink, err := AWS.GetVPCLinkByTag(clusterconfig.ClusterNameTag, Cluster.ClusterName)
+if err != nil {
+return err
+} else if vpcLink == nil {
+return ErrorNoVPCLink()
+}
+Cluster.VPCLink = vpcLink
+
+integration, err := AWS.GetVPCLinkIntegration(*Cluster.APIGateway.ApiId, *Cluster.VPCLink.VpcLinkId)
+if err != nil {
+return err
+} else if integration == nil {
+return ErrorNoVPCLinkIntegration()
+}
+Cluster.VPCLinkIntegration = integration
 }
-Cluster.VPCLinkIntegration = integration
 }
 
 Cluster.InstanceMetadata = aws.InstanceMetadatas[*Cluster.Region][*Cluster.InstanceType]

pkg/operator/operator/gateway.go

@@ -27,6 +27,10 @@ import (
 )
 
 func AddAPIToAPIGateway(endpoint string, apiGatewayType userconfig.APIGatewayType) error {
+if config.Cluster.APIGateway == nil {
+return nil
+}
+
 if apiGatewayType == userconfig.NoneAPIGatewayType {
 return nil
 }
@@ -71,6 +75,10 @@ func AddAPIToAPIGateway(endpoint string, apiGatewayType userconfig.APIGatewayTyp
 }
 
 func RemoveAPIFromAPIGateway(endpoint string, apiGatewayType userconfig.APIGatewayType) error {
+if config.Cluster.APIGateway == nil {
+return nil
+}
+
 if apiGatewayType == userconfig.NoneAPIGatewayType {
 return nil
 }
@@ -102,6 +110,10 @@ func UpdateAPIGateway(
 newAPIGatewayType userconfig.APIGatewayType,
 ) error {
 
+if config.Cluster.APIGateway == nil {
+return nil
+}
+
 if prevAPIGatewayType == userconfig.NoneAPIGatewayType && newAPIGatewayType == userconfig.NoneAPIGatewayType {
 return nil
 }

pkg/operator/resources/apisplitter/api.go

@@ -160,7 +160,7 @@ func areVirtualServiceEqual(vs1, vs2 *istioclientnetworking.VirtualService) bool
 
 // APIBaseURL returns BaseURL of the API without resource endpoint
 func APIBaseURL(api *spec.API) (string, error) {
-if api.Networking.APIGateway == userconfig.PublicAPIGatewayType {
+if api.Networking.APIGateway == userconfig.PublicAPIGatewayType && config.Cluster.APIGateway != nil {
 return *config.Cluster.APIGateway.ApiEndpoint, nil
 }
 return operator.APILoadBalancerURL()

pkg/operator/resources/errors.go

@@ -21,6 +21,7 @@ import (
 
 "github.com/cortexlabs/cortex/pkg/lib/errors"
 "github.com/cortexlabs/cortex/pkg/lib/strings"
+s "github.com/cortexlabs/cortex/pkg/lib/strings"
 "github.com/cortexlabs/cortex/pkg/types/userconfig"
 )
 
@@ -31,6 +32,7 @@ const (
 ErrNoAvailableNodeComputeLimit = "resources.no_available_node_compute_limit"
 ErrAPIUsedByAPISplitter = "resources.syncapi_used_by_apisplitter"
 ErrNotDeployedAPIsAPISplitter = "resources.trafficsplit_apis_not_deployed"
+ErrAPIGatewayDisabled = "resources.api_gateway_disabled"
 )
 
 func ErrorOperationNotSupportedForKind(kind userconfig.Kind) error {
@@ -78,3 +80,15 @@ func ErrorNotDeployedAPIsAPISplitter(notDeployedAPIs []string) error {
 Message: fmt.Sprintf("unable to find specified %s: %s", strings.PluralS("api", len(notDeployedAPIs)), strings.StrsAnd(notDeployedAPIs)),
 })
 }
+
+func ErrorAPIGatewayDisabled(apiGatewayType userconfig.APIGatewayType) error {
+msg := fmt.Sprintf("%s is not permitted because api gateway is disabled cluster-wide", s.UserStr(apiGatewayType))
+if apiGatewayType == userconfig.PublicAPIGatewayType {
+msg += fmt.Sprintf(" (%s is the default value, and the valid values are %s)", s.UserStr(userconfig.PublicAPIGatewayType), s.UserStrsAnd(userconfig.APIGatewayTypeStrings()))
+}
+
+return errors.WithStack(&errors.Error{
+Kind: ErrAPIGatewayDisabled,
+Message: msg,
+})
+}