Add cluster name to s3 prefix for all objects

cli/cmd/cluster.go

@@ -183,7 +183,7 @@ var _upCmd = &cobra.Command{
 exit.Error(err)
 }
 
-err = createLogGroupIfNotFound(awsClient, clusterConfig.LogGroup, clusterConfig.Tags)
+err = createLogGroupIfNotFound(awsClient, clusterConfig.ClusterName, clusterConfig.Tags)
 if err != nil {
 exit.Error(err)
 }
@@ -659,7 +659,7 @@ var _exportCmd = &cobra.Command{
 exit.Error(err)
 }
 
-err = awsClient.DownloadFileFromS3(info.ClusterConfig.Bucket, apiSpec.RawAPIKey(), path.Join(baseDir, apiSpec.FileName))
+err = awsClient.DownloadFileFromS3(info.ClusterConfig.Bucket, apiSpec.RawAPIKey(info.ClusterConfig.ClusterName), path.Join(baseDir, apiSpec.FileName))
 if err != nil {
 exit.Error(err)
 }

cli/cmd/lib_cluster_config.go

@@ -276,11 +276,6 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 }
 userClusterConfig.Bucket = cachedClusterConfig.Bucket
 
-if userClusterConfig.LogGroup != "" && userClusterConfig.LogGroup != cachedClusterConfig.LogGroup {
-return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.LogGroupKey, cachedClusterConfig.LogGroup)
-}
-userClusterConfig.LogGroup = cachedClusterConfig.LogGroup
-
 if userClusterConfig.InstanceType != nil && *userClusterConfig.InstanceType != *cachedClusterConfig.InstanceType {
 return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.InstanceTypeKey, *cachedClusterConfig.InstanceType)
 }
@@ -343,6 +338,66 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 }
 userClusterConfig.APIGatewaySetting = cachedClusterConfig.APIGatewaySetting
 
+if s.Obj(cachedClusterConfig.VPCCIDR) != s.Obj(userClusterConfig.VPCCIDR) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.VPCCIDRKey, cachedClusterConfig.VPCCIDR)
+}
+userClusterConfig.VPCCIDR = cachedClusterConfig.VPCCIDR
+
+if s.Obj(cachedClusterConfig.ImageDownloader) != s.Obj(userClusterConfig.ImageDownloader) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageDownloaderKey, cachedClusterConfig.ImageDownloader)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageRequestMonitor) != s.Obj(userClusterConfig.ImageRequestMonitor) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageRequestMonitorKey, cachedClusterConfig.ImageRequestMonitor)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageClusterAutoscaler) != s.Obj(userClusterConfig.ImageClusterAutoscaler) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageClusterAutoscalerKey, cachedClusterConfig.ImageClusterAutoscaler)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageMetricsServer) != s.Obj(userClusterConfig.ImageMetricsServer) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageMetricsServerKey, cachedClusterConfig.ImageMetricsServer)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageInferentia) != s.Obj(userClusterConfig.ImageInferentia) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageInferentiaKey, cachedClusterConfig.ImageInferentia)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageNeuronRTD) != s.Obj(userClusterConfig.ImageNeuronRTD) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageNeuronRTDKey, cachedClusterConfig.ImageNeuronRTD)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageNvidia) != s.Obj(userClusterConfig.ImageNvidia) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageNvidiaKey, cachedClusterConfig.ImageNvidia)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageFluentd) != s.Obj(userClusterConfig.ImageFluentd) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageFluentdKey, cachedClusterConfig.ImageFluentd)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageStatsd) != s.Obj(userClusterConfig.ImageStatsd) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageStatsdKey, cachedClusterConfig.ImageStatsd)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageIstioProxy) != s.Obj(userClusterConfig.ImageIstioProxy) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageIstioProxyKey, cachedClusterConfig.ImageIstioProxy)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
+if s.Obj(cachedClusterConfig.ImageIstioPilot) != s.Obj(userClusterConfig.ImageIstioPilot) {
+return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageIstioPilotKey, cachedClusterConfig.ImageIstioPilot)
+}
+userClusterConfig.ImageDownloader = cachedClusterConfig.ImageDownloader
+
 if userClusterConfig.Spot != nil && *userClusterConfig.Spot != *cachedClusterConfig.Spot {
 return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.SpotKey, *cachedClusterConfig.Spot)
 }
@@ -384,14 +439,8 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 return errors.Wrap(clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.OnDemandBackupKey, cachedClusterConfig.SpotConfig.OnDemandBackup), clusterconfig.SpotConfigKey)
 }
 }
-
 userClusterConfig.SpotConfig = cachedClusterConfig.SpotConfig
 
-if s.Obj(cachedClusterConfig.VPCCIDR) != s.Obj(userClusterConfig.VPCCIDR) {
-return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.VPCCIDRKey, cachedClusterConfig.VPCCIDR)
-}
-userClusterConfig.VPCCIDR = cachedClusterConfig.VPCCIDR
-
 return nil
 }
 
@@ -490,7 +539,7 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds A
 if clusterConfig.SubnetVisibility == clusterconfig.PrivateSubnetVisibility {
 privateSubnetMsg = ", and will use private subnets for all EC2 instances"
 }
-fmt.Printf("cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\n\n", clusterConfig.Bucket, clusterConfig.LogGroup, privateSubnetMsg)
+fmt.Printf("cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\n\n", clusterConfig.Bucket, clusterConfig.ClusterName, privateSubnetMsg)
 
 if clusterConfig.APIGatewaySetting == clusterconfig.NoneAPIGatewaySetting {
 fmt.Print("warning: you've disabled API Gateway cluster-wide, so APIs will not be able to create API Gateway endpoints (they will still be reachable via the API load balancer; see https://docs.cortex.dev/deployments/networking for more information)\n\n")
@@ -538,9 +587,6 @@ func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds A
 }
 items.Add(clusterconfig.BucketUserKey, clusterConfig.Bucket)
 items.Add(clusterconfig.ClusterNameUserKey, clusterConfig.ClusterName)
-if clusterConfig.LogGroup != defaultConfig.LogGroup {
-items.Add(clusterconfig.LogGroupUserKey, clusterConfig.LogGroup)
-}
 
 items.Add(clusterconfig.InstanceTypeUserKey, *clusterConfig.InstanceType)
 items.Add(clusterconfig.MinInstancesUserKey, *clusterConfig.MinInstances)

cli/local/api.go

@@ -70,7 +70,7 @@ func UpdateAPI(apiConfig *userconfig.API, configPath string, projectID string, d
 return nil, "", err
 }
 
-newAPISpec := spec.GetAPISpec(apiConfig, projectID, _deploymentID)
+newAPISpec := spec.GetAPISpec(apiConfig, projectID, _deploymentID, "")
 
 // apiConfig.Predictor.ModelPath was already added to apiConfig.Predictor.Models for ease of use
 if len(apiConfig.Predictor.Models) > 0 {

docs/cluster-management/config.md

@@ -63,9 +63,6 @@ operator_load_balancer_scheme: internet-facing # must be "internet-facing" or "
 # if set to "none", no APIs will be allowed to use API Gateway
 api_gateway: public # must be "public" or "none"
 
-# CloudWatch log group for cortex (default: <cluster_name>)
-log_group: cortex
-
 # additional tags to assign to aws resources for labelling and cost allocation (by default, all resources will be tagged with cortex.dev/cluster-name=<cluster_name>)
 tags: # <string>: <string> map of key/value pairs
 

docs/cluster-management/uninstall.md

@@ -39,8 +39,8 @@ aws s3 ls
 # delete the S3 bucket
 aws s3 rb --force s3://<bucket>
 
-# delete the log group (replace <log_group> with what was configured during installation, default: cortex)
-aws logs describe-log-groups --log-group-name-prefix=<log_group> --query logGroups[*].[logGroupName] --output text | xargs -I {} aws logs delete-log-group --log-group-name {}
+# delete the log group (replace <cluster_name> with the name of your cluster, default: cortex)
+aws logs describe-log-groups --log-group-name-prefix=<cluster_name> --query logGroups[*].[logGroupName] --output text | xargs -I {} aws logs delete-log-group --log-group-name {}
 ```
 
 If you've configured a custom domain for your APIs, you may wish to remove the SSL Certificate and Hosted Zone for the domain by following these [instructions](../guides/custom-domain.md#cleanup).

docs/contributing/development.md

@@ -139,7 +139,6 @@ min_instances: 2
 max_instances: 5
 bucket: cortex-cluster-<your_name>
 region: us-west-2
-log_group: cortex
 cluster_name: cortex
 
 image_operator: XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/operator:latest