Validate # of security groups and in/out rules before clustering up

cli/cmd/lib_cluster_config.go

@@ -135,7 +135,7 @@ func getInstallClusterConfig(awsClient *aws.Client, clusterConfigFile string, di
 return nil, err
 }
 
-err = clusterConfig.Validate(awsClient, false)
+err = clusterConfig.Validate(awsClient)
 if err != nil {
 err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortex.dev/v/%s/", consts.CortexVersionMinor))
 return nil, errors.Wrap(err, clusterConfigFile)

pkg/lib/aws/ec2.go

@@ -334,6 +334,28 @@ func (c *Client) DescribeVpcs() ([]ec2.Vpc, error) {
 return vpcs, nil
 }
 
+func (c *Client) DescribeSecurityGroups() ([]ec2.SecurityGroup, error) {
+var sgs []ec2.SecurityGroup
+err := c.EC2().DescribeSecurityGroupsPages(&ec2.DescribeSecurityGroupsInput{}, func(output *ec2.DescribeSecurityGroupsOutput, lastPage bool) bool {
+if output == nil {
+return false
+}
+for _, sg := range output.SecurityGroups {
+if sg == nil {
+continue
+}
+sgs = append(sgs, *sg)
+}
+
+return true
+})
+if err != nil {
+return nil, errors.WithStack(err)
+}
+
+return sgs, nil
+}
+
 func (c *Client) ListVolumes(tags ...ec2.Tag) ([]ec2.Volume, error) {
 var volumes []ec2.Volume
 err := c.EC2().DescribeVolumesPages(&ec2.DescribeVolumesInput{}, func(output *ec2.DescribeVolumesOutput, lastPage bool) bool {

pkg/lib/aws/errors.go

@@ -46,6 +46,8 @@ const (
 ErrEIPLimitExceeded = "aws.eip_limit_exceeded"
 ErrInternetGatewayLimitExceeded = "aws.internet_gateway_limit_exceeded"
 ErrVPCLimitExceeded = "aws.vpc_limit_exceeded"
+ErrSecurityGroupRulesExceeded = "aws.security_group_rules_exceeded"
+ErrSecurityGroupLimitExceeded = "aws.security_group_limit_exceeded"
 )
 
 func IsAWSError(err error) bool {
@@ -232,3 +234,19 @@ func ErrorVPCLimitExceeded(currentLimit, additionalQuotaRequired int, region str
 Message: fmt.Sprintf("VPC limit of %d exceeded in region %s; remove some of the existing VPCs or increase your quota for VPCs by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url),
 })
 }
+
+func ErrorSecurityGroupRulesExceeded(currentLimit, additionalQuotaRequired int, region string) error {
+url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas"
+return errors.WithStack(&errors.Error{
+Kind: ErrSecurityGroupRulesExceeded,
+Message: fmt.Sprintf("security group rules limit of %d exceeded in region %s; use fewer availability zones, remove some node groups from your cluster config, reduce the number of CIDR white lists (if you have any), or increase your quota for inbound/outbound rules per security group by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url),
+})
+}
+
+func ErrorSecurityGroupLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error {
+url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas"
+return errors.WithStack(&errors.Error{
+Kind: ErrSecurityGroupLimitExceeded,
+Message: fmt.Sprintf("security group limit of %d exceeded in region %s; remove some node groups from your cluster config or increase your quota for security groups by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url),
+})
+}

pkg/lib/aws/servicequotas.go

@@ -32,10 +32,18 @@ var _standardInstanceCategories = strset.New("a", "c", "d", "h", "i", "m", "r",
 var _knownInstanceCategories = strset.Union(_standardInstanceCategories, strset.New("p", "g", "inf", "x", "f"))
 
 const (
-_elasticIPsQuotaCode = "L-0263D0A3"
-_internetGatewayQuotaCode = "L-A4707A72"
-_natGatewayQuotaCode = "L-FE5A380F"
-_vpcQuotaCode = "L-F678F1CE"
+_elasticIPsQuotaCode = "L-0263D0A3"
+_internetGatewayQuotaCode = "L-A4707A72"
+_natGatewayQuotaCode = "L-FE5A380F"
+_vpcQuotaCode = "L-F678F1CE"
+_securityGroupsQuotaCode = "L-E79EC296"
+_securityGroupRulesQuotaCode = "L-0EA8095F"
+
+// 11 inbound rules
+_baseInboundRulesForNodeGroup = 11
+_inboundRulesPerAZ = 8
+// ClusterSharedNodeSecurityGroup, ControlPlaneSecurityGroup, eks-cluster-sg-<cluster-name>, and operator security group
+_baseNumberOfSecurityGroups = 4
 )
 
 type InstanceTypeRequests struct {
@@ -145,12 +153,21 @@ func (c *Client) VerifyInstanceQuota(instances []InstanceTypeRequests) error {
 return nil
 }
 
-func (c *Client) VerifyNetworkQuotas(requiredInternetGateways int, natGatewayRequired bool, highlyAvailableNATGateway bool, requiredVPCs int, availabilityZones strset.Set) error {
+func (c *Client) VerifyNetworkQuotas(
+requiredInternetGateways int,
+natGatewayRequired bool,
+highlyAvailableNATGateway bool,
+requiredVPCs int,
+availabilityZones strset.Set,
+numNodeGroups int,
+longestCIDRWhiteList int) error {
 quotaCodeToValueMap := map[string]int{
-_elasticIPsQuotaCode: 0, // elastic IP quota code
-_internetGatewayQuotaCode: 0, // internet gw quota code
-_natGatewayQuotaCode: 0, // nat gw quota code
-_vpcQuotaCode: 0, // vpc quota code
+_elasticIPsQuotaCode: 0, // elastic IP quota code
+_internetGatewayQuotaCode: 0, // internet gw quota code
+_natGatewayQuotaCode: 0, // nat gw quota code
+_vpcQuotaCode: 0, // vpc quota code
+_securityGroupsQuotaCode: 0, // security groups quota code
+_securityGroupRulesQuotaCode: 0, // security group rules quota code
 }
 
 err := c.ServiceQuotas().ListServiceQuotasPages(
@@ -285,5 +302,52 @@ func (c *Client) VerifyNetworkQuotas(requiredInternetGateways int, natGatewayReq
 }
 }
 
+// check rules quota for nodegroup SGs
+requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)
+if requiredRulesForSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
+additionalQuotaRequired := requiredRulesForSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
+return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
+}
+
+// check rules quota for control plane SG
+requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)
+if requiredRulesForCPSG > quotaCodeToValueMap[_securityGroupRulesQuotaCode] {
+additionalQuotaRequired := requiredRulesForCPSG - quotaCodeToValueMap[_securityGroupRulesQuotaCode]
+return ErrorSecurityGroupRulesExceeded(quotaCodeToValueMap[_securityGroupRulesQuotaCode], additionalQuotaRequired, c.Region)
+}
+
+// check security groups quota
+requiredSecurityGroups := requiredSecurityGroups(numNodeGroups)
+sgs, err := c.DescribeSecurityGroups()
+if err != nil {
+return err
+}
+if quotaCodeToValueMap[_securityGroupsQuotaCode]-len(sgs)-requiredSecurityGroups < 0 {
+additionalQuotaRequired := len(sgs) + requiredSecurityGroups - quotaCodeToValueMap[_securityGroupsQuotaCode]
+return ErrorSecurityGroupLimitExceeded(quotaCodeToValueMap[_securityGroupsQuotaCode], additionalQuotaRequired, c.Region)
+
+}
+
 return nil
 }
+
+func requiredRulesForNodeGroupSecurityGroup(numAZs, whitelistLength int) int {
+whitelistRuleCount := 0
+if whitelistLength == 1 {
+whitelistRuleCount = 1
+} else if whitelistLength > 1 {
+whitelistRuleCount = 1 + 5*(whitelistLength-1)
+}
+return _baseInboundRulesForNodeGroup + numAZs*_inboundRulesPerAZ + whitelistRuleCount
+}
+
+func requiredRulesForControlPlaneSecurityGroup(numNodeGroups int) int {
+// +1 for the operator node group
+// this is the number of outbound rules (there are half as many inbound rules, so that is not the limiting factor)
+return 2 * (numNodeGroups + 1)
+}
+
+func requiredSecurityGroups(numNodeGroups int) int {
+// each node group requires a security group
+return _baseNumberOfSecurityGroups + numNodeGroups
+}

pkg/types/clusterconfig/cluster_config.go

@@ -781,7 +781,7 @@ func (cc *CoreConfig) SQSNamePrefix() string {
 }
 
 // this validates the user-provided cluster config
-func (cc *Config) Validate(awsClient *aws.Client, skipQuotaVerification bool) error {
+func (cc *Config) Validate(awsClient *aws.Client) error {
 fmt.Print("verifying your configuration ...\n\n")
 
 numNodeGroups := len(cc.NodeGroups)
@@ -817,12 +817,10 @@ func (cc *Config) Validate(awsClient *aws.Client, skipQuotaVerification bool) er
 })
 }
 
-if !skipQuotaVerification {
-if err := awsClient.VerifyInstanceQuota(instances); err != nil {
-// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
-if !aws.IsAWSError(err) {
-return errors.Wrap(err, NodeGroupsKey)
-}
+if err := awsClient.VerifyInstanceQuota(instances); err != nil {
+// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
+if !aws.IsAWSError(err) {
+return errors.Wrap(err, NodeGroupsKey)
 }
 }
 
@@ -909,16 +907,15 @@ func (cc *Config) Validate(awsClient *aws.Client, skipQuotaVerification bool) er
 }
 }
 
-if !skipQuotaVerification {
-var requiredVPCs int
-if len(cc.Subnets) == 0 {
-requiredVPCs = 1
-}
-if err := awsClient.VerifyNetworkQuotas(1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones)); err != nil {
-// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
-if !aws.IsAWSError(err) {
-return err
-}
+var requiredVPCs int
+if len(cc.Subnets) == 0 {
+requiredVPCs = 1
+}
+longestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList))
+if err := awsClient.VerifyNetworkQuotas(1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), longestCIDRWhiteList); err != nil {
+// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API
+if !aws.IsAWSError(err) {
+return err
 }
 }