Separate operator workload nodegroup

Makefile

@@ -25,7 +25,7 @@ devstart:
 @./dev/operator_local.sh || true
 
 kubectl:
-@eval $$(python ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+@eval $$(python ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eksctl utils write-kubeconfig --name="$$CORTEX_CLUSTER_NAME" | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
 @kubectl config set-context --current --namespace=cortex >/dev/null
 
 cluster-up:

cli/cmd/lib_manager.go

@@ -124,14 +124,15 @@ func runManagerCommand(entrypoint string, clusterConfig *clusterconfig.ClusterCo
 if err != nil {
 return "", errors.WithStack(err)
 }
+
 if err := files.WriteFile(clusterConfigBytes, cachedClusterConfigPath); err != nil {
 return "", err
 }
 
 containerConfig := &container.Config{
 Image: clusterConfig.ImageManager,
 Entrypoint: []string{"/bin/bash", "-c"},
-Cmd: []string{"sleep 0.1 && eval $(python /root/cluster_config_env.py /.cortex/cluster.yaml) && " + entrypoint},
+Cmd: []string{"sleep 0.1 && eval $(python /root/instance_metadata.py /.cortex/cluster.yaml) && eval $(python /root/cluster_config_env.py /.cortex/cluster.yaml) && " + entrypoint},
 Tty: true,
 AttachStdout: true,
 AttachStderr: true,

dev/operator_local.sh

@@ -22,6 +22,10 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 export CORTEX_OPERATOR_IN_CLUSTER=false
 export CORTEX_CLUSTER_CONFIG_PATH=$ROOT/dev/config/cluster.yaml
 
+pip3 install -r $ROOT/manager/requirements.txt
+
+python3 $ROOT/manager/instance_metadata.py $CORTEX_CLUSTER_CONFIG_PATH
+
 kill $(pgrep -f rerun) >/dev/null 2>&1 || true
 updated_cli_config=$(cat $HOME/.cortex/default.json | jq '.cortex_url = "http://localhost:8888"') && echo $updated_cli_config > $HOME/.cortex/default.json
 rerun -watch $ROOT/pkg $ROOT/cli $ROOT/dev/config -ignore $ROOT/vendor $ROOT/bin -run sh -c \

images/manager/Dockerfile

@@ -8,6 +8,10 @@ RUN pip install --upgrade pip && \
  pip install awscli --upgrade --user && \
  rm -rf /root/.cache/pip*
 
+COPY manager/requirements.txt /root/requirements.txt
+
+RUN pip install -r /root/requirements.txt
+
 RUN apk add --no-cache bash curl gettext jq openssl
 
 RUN curl --location "https://github.com/weaveworks/eksctl/releases/download/0.8.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \

manager/eks.yaml

@@ -21,15 +21,46 @@ metadata:
  version: "1.14"
 
 nodeGroups:
- - name: ng-1
+ - name: ng-cortex-operator
+ instanceType: t3.medium
+ minSize: 2
+ maxSize: 2
+ desiredCapacity: 2
+ ami: auto
+ iam:
+ withAddonPolicies:
+ autoScaler: true
+ kubeletExtraConfig:
+ kubeReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ kubeReservedCgroup: /kube-reserved
+ systemReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ evictionHard:
+ memory.available: 200Mi
+ nodefs.available: 5%
+
+ - name: ng-cortex-worker
  instanceType: $CORTEX_INSTANCE_TYPE
  minSize: $CORTEX_MIN_INSTANCES
  maxSize: $CORTEX_MAX_INSTANCES
- desiredCapacity: $CORTEX_MIN_INSTANCES
+ desiredCapacity: $CORTEX_DESIRED_INSTANCES
  ami: auto
  iam:
  withAddonPolicies:
  autoScaler: true
+ tags:
+ k8s.io/cluster-autoscaler/enabled: 'true'
+ k8s.io/cluster-autoscaler/node-template/label/workload: 'true'
+ labels:
+ lifecycle: Ec2Spot
+ workload: "true"
+ taints:
+ workload: "true:NoSchedule"
  kubeletExtraConfig:
  kubeReserved:
  cpu: 150m
@@ -41,5 +72,5 @@ nodeGroups:
  memory: 300Mi
  ephemeral-storage: 1Gi
  evictionHard:
- memory.available:  200Mi
+ memory.available: 200Mi
  nodefs.available: 5%

manager/eks_gpu.yaml

@@ -0,0 +1,79 @@
+# Copyright 2019 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+ name: $CORTEX_CLUSTER_NAME
+ region: $CORTEX_REGION
+ version: "1.14"
+
+nodeGroups:
+ - name: ng-cortex-operator
+ instanceType: t3.medium
+ minSize: 2
+ maxSize: 2
+ desiredCapacity: 2
+ ami: auto
+ iam:
+ withAddonPolicies:
+ autoScaler: true
+ kubeletExtraConfig:
+ kubeReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ kubeReservedCgroup: /kube-reserved
+ systemReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ evictionHard:
+ memory.available: 200Mi
+ nodefs.available: 5%
+
+ - name: ng-cortex-worker
+ instanceType: $CORTEX_INSTANCE_TYPE
+ minSize: $CORTEX_MIN_INSTANCES
+ maxSize: $CORTEX_MAX_INSTANCES
+ desiredCapacity: $CORTEX_DESIRED_INSTANCES
+ ami: auto
+ iam:
+ withAddonPolicies:
+ autoScaler: true
+ tags:
+ k8s.io/cluster-autoscaler/enabled: 'true'
+ k8s.io/cluster-autoscaler/node-template/label/nvidia.com/gpu: 'true'
+ k8s.io/cluster-autoscaler/node-template/taint/dedicated: nvidia.com/gpu=true
+ labels:
+ lifecycle: Ec2Spot
+ workload: "true"
+ nvidia.com/gpu: 'true'
+ taints:
+ nvidia.com/gpu: "true:NoSchedule"
+ workload: "true:NoSchedule"
+ kubeletExtraConfig:
+ kubeReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ kubeReservedCgroup: /kube-reserved
+ systemReserved:
+ cpu: 150m
+ memory: 300Mi
+ ephemeral-storage: 1Gi
+ evictionHard:
+ memory.available: 200Mi
+ nodefs.available: 5%

manager/install.sh

@@ -32,8 +32,18 @@ function ensure_eks() {
  exit 1
  fi
 
+ if [ $CORTEX_MIN_INSTANCES -lt 1 ]; then
+ CORTEX_DESIRED_INSTANCES=1
+ else
+ CORTEX_DESIRED_INSTANCES=$CORTEX_MIN_INSTANCES
+ fi
+
  echo -e "￮ Spinning up the cluster ... (this will take about 15 minutes)\n"
- envsubst < eks.yaml | eksctl create cluster -f -
+ if [ $CORTEX_INSTANCE_GPU -ne 0 ]; then
+ envsubst < eks_gpu.yaml | eksctl create cluster -f -
+ else
+ envsubst < eks.yaml | eksctl create cluster -f -
+ fi
  echo -e "\n✓ Spun up the cluster"
  return
  fi
@@ -60,15 +70,15 @@ function ensure_eks() {
  echo "✓ Cluster is running"
 
  # Check if instance type changed
- ng_info=$(eksctl get nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --name ng-1 -o json)
- ng_instance_type=$(echo "$ng_info" | jq -r ".[] | select( .Cluster == \"$CORTEX_CLUSTER_NAME\" ) | select( .Name == \"ng-1\" ) | .InstanceType")
+ ng_info=$(eksctl get nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --name ng-cortex-worker -o json)
+ ng_instance_type=$(echo "$ng_info" | jq -r ".[] | select( .Cluster == \"$CORTEX_CLUSTER_NAME\" ) | select( .Name == \"ng-cortex-worker\" ) | .InstanceType")
  if [ "$ng_instance_type" != "$CORTEX_INSTANCE_TYPE" ]; then
  echo -e "\nerror: Cortex does not currently support changing the instance type of a running cluster; please run \`cortex cluster down\` followed by \`cortex cluster up\` to create a new cluster"
  exit 1
  fi
 
  # Check for change in min/max instances
- asg_info=$(aws autoscaling describe-auto-scaling-groups --region $CORTEX_REGION --query 'AutoScalingGroups[?contains(Tags[?Key==`alpha.eksctl.io/nodegroup-name`].Value, `ng-1`)]')
+ asg_info=$(aws autoscaling describe-auto-scaling-groups --region $CORTEX_REGION --query 'AutoScalingGroups[?contains(Tags[?Key==`alpha.eksctl.io/nodegroup-name`].Value, `ng-cortex-worker`)]')
  asg_name=$(echo "$asg_info" | jq -r 'first | .AutoScalingGroupName')
  asg_min_size=$(echo "$asg_info" | jq -r 'first | .MinSize')
  asg_max_size=$(echo "$asg_info" | jq -r 'first | .MaxSize')

manager/instance_metadata.py

@@ -0,0 +1,79 @@
+import requests
+import sys
+import re
+import os
+import pathlib
+import json
+import yaml
+
+PRICING_ENDPOINT_TEMPLATE = (
+ "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/{}/index.json"
+)
+
+
+def download_metadata(cluster_config):
+ response = requests.get(PRICING_ENDPOINT_TEMPLATE.format(cluster_config["region"]))
+ offers = response.json()
+
+ instance_mapping = {}
+
+ for product_id, product in offers["products"].items():
+ if product.get("attributes") is None:
+ continue
+ if product["attributes"].get("servicecode") != "AmazonEC2":
+ continue
+ if product["attributes"].get("tenancy") != "Shared":
+ continue
+ if product["attributes"].get("operatingSystem") != "Linux":
+ continue
+ if product["attributes"].get("capacitystatus") != "Used":
+ continue
+ if product["attributes"].get("operation") != "RunInstances":
+ continue
+ price_dimensions = list(offers["terms"]["OnDemand"][product["sku"]].values())[0][
+ "priceDimensions"
+ ]
+
+ price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"]
+
+ instance_type = product["attributes"]["instanceType"]
+ metadata = {
+ "sku": product["sku"],
+ "instance_type": instance_type,
+ "cpu": int(product["attributes"]["vcpu"]),
+ "mem": int(
+ float(re.sub("[^0-9\\.]", "", product["attributes"]["memory"].split(" ")[0])) * 1024
+ ),
+ "price": float(price),
+ }
+ if product["attributes"].get("gpu") is not None:
+ metadata["gpu"] = product["attributes"]["gpu"]
+ instance_mapping[instance_type] = metadata
+
+ return instance_mapping
+
+
+def get_metadata(cluster_config):
+ return download_metadata(cluster_config)
+
+
+def set_ec2_metadata(cluster_config_path):
+ with open(cluster_config_path, "r") as cluster_config_file:
+ cluster_config = yaml.safe_load(cluster_config_file)
+ instance_mapping = get_metadata(cluster_config)
+ instance_type = instance_mapping[cluster_config["instance_type"]]
+
+ cluster_config["instance_mem"] = str(instance_type["mem"]) + "Mi"
+ cluster_config["instance_cpu"] = str(instance_type["cpu"])
+ cluster_config["instance_gpu"] = int(instance_type.get("gpu", 0))
+
+ with open(cluster_config_path, "w") as cluster_config_file:
+ yaml.dump(cluster_config, cluster_config_file, default_flow_style=False)
+
+
+def main():
+ set_ec2_metadata(sys.argv[1])
+
+
+if __name__ == "__main__":
+ main()

manager/manifests/cluster-autoscaler.yaml

@@ -32,40 +32,48 @@ metadata:
  k8s-addon: cluster-autoscaler.addons.k8s.io
  k8s-app: cluster-autoscaler
 rules:
-- apiGroups: [""]
- resources: ["events","endpoints"]
- verbs: ["create", "patch"]
-- apiGroups: [""]
- resources: ["pods/eviction"]
- verbs: ["create"]
-- apiGroups: [""]
- resources: ["pods/status"]
- verbs: ["update"]
-- apiGroups: [""]
- resources: ["endpoints"]
- resourceNames: ["cluster-autoscaler"]
- verbs: ["get","update"]
-- apiGroups: [""]
- resources: ["nodes"]
- verbs: ["watch","list","get","update"]
-- apiGroups: [""]
- resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"]
- verbs: ["watch","list","get"]
-- apiGroups: ["extensions"]
- resources: ["replicasets","daemonsets"]
- verbs: ["watch","list","get"]
-- apiGroups: ["policy"]
- resources: ["poddisruptionbudgets"]
- verbs: ["watch","list"]
-- apiGroups: ["apps"]
- resources: ["statefulsets", "replicasets", "daemonsets"]
- verbs: ["watch","list","get"]
-- apiGroups: ["storage.k8s.io"]
- resources: ["storageclasses"]
- verbs: ["watch","list","get"]
-- apiGroups: ["batch", "extensions"]
- resources: ["jobs"]
- verbs: ["get", "list", "watch", "patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["csinodes"]
+ verbs: ["watch", "list", "get"]
+ - apiGroups: [""]
+ resources: ["events", "endpoints"]
+ verbs: ["create", "patch"]
+ - apiGroups: [""]
+ resources: ["pods/eviction"]
+ verbs: ["create"]
+ - apiGroups: [""]
+ resources: ["pods/status"]
+ verbs: ["update"]
+ - apiGroups: [""]
+ resources: ["endpoints"]
+ resourceNames: ["cluster-autoscaler"]
+ verbs: ["get", "update"]
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["watch", "list", "get", "update"]
+ - apiGroups: [""]
+ resources:
+ - "pods"
+ - "services"
+ - "replicationcontrollers"
+ - "persistentvolumeclaims"
+ - "persistentvolumes"
+ verbs: ["watch", "list", "get"]
+ - apiGroups: ["extensions"]
+ resources: ["replicasets", "daemonsets"]
+ verbs: ["watch", "list", "get"]
+ - apiGroups: ["policy"]
+ resources: ["poddisruptionbudgets"]
+ verbs: ["watch", "list"]
+ - apiGroups: ["apps"]
+ resources: ["statefulsets", "replicasets", "daemonsets"]
+ verbs: ["watch", "list", "get"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["storageclasses"]
+ verbs: ["watch", "list", "get"]
+ - apiGroups: ["batch", "extensions"]
+ resources: ["jobs"]
+ verbs: ["get", "list", "watch", "patch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1