Separate prometheus and operator workloads

manager/generate_eks.py

@@ -305,20 +305,38 @@ def generate_eks(
  return
 
  operator_nodegroup = default_nodegroup(cluster_config)
+ # TODO validate requests when clustering up
  operator_settings = {
  "ami": get_ami(ami_map, "t3.medium"),
  "name": "cx-operator",
  "instanceType": "t3.medium",
- "minSize": 2,
- "maxSize": 2,
- "desiredCapacity": 2,
+ "minSize": 1,
+ "maxSize": 25,
+ "desiredCapacity": 1,
  "volumeType": "gp3",
  "volumeSize": 20,
  "volumeIOPS": 3000,
  "volumeThroughput": 125,
  }
  operator_nodegroup = merge_override(operator_nodegroup, operator_settings)
 
+ prometheus_nodegroup = default_nodegroup(cluster_config)
+ prometheus_settings = {
+ "ami": get_ami(ami_map, "t3.xlarge"),
+ "name": "cx-prometheus",
+ "instanceType": "t3.xlarge",
+ "minSize": 1,
+ "maxSize": 1,
+ "desiredCapacity": 1,
+ "volumeType": "gp3",
+ "volumeSize": 20,
+ "volumeIOPS": 3000,
+ "volumeThroughput": 125,
+ "labels": {"prometheus": "true"},
+ "taints": {"prometheus": "true:NoSchedule"},
+ }
+ prometheus_nodegroup = merge_override(prometheus_nodegroup, prometheus_settings)
+
  worker_nodegroups = get_all_worker_nodegroups(ami_map, cluster_config)
 
  nat_gateway = "Disable"
@@ -337,7 +355,7 @@ def generate_eks(
  "tags": cluster_config["tags"],
  },
  "vpc": {"nat": {"gateway": nat_gateway}},
- "nodeGroups": [operator_nodegroup] + worker_nodegroups,
+ "nodeGroups": [operator_nodegroup, prometheus_nodegroup] + worker_nodegroups,
  "addons": [
  {
  "name": "vpc-cni",

manager/manifests/cluster-autoscaler.yaml.j2

@@ -169,11 +169,11 @@ spec:
  name: cluster-autoscaler
  resources:
  limits:
- cpu: 100m
- memory: 300Mi
+ cpu: 300m
+ memory: 1Gi
  requests:
  cpu: 100m
- memory: 300Mi
+ memory: 200Mi
  command:
  - ./cluster-autoscaler
  - --v=4

manager/manifests/fluent-bit.yaml.j2

@@ -249,3 +249,6 @@ spec:
  - key: workload
  operator: Exists
  effect: NoSchedule
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule

manager/manifests/grafana/grafana.yaml

@@ -173,6 +173,12 @@ spec:
  - name: grafana-dashboard-nodes
  configMap:
  name: grafana-dashboard-nodes
+ nodeSelector:
+ prometheus: "true"
+ tolerations:
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
  affinity:
  podAffinity:
  preferredDuringSchedulingIgnoredDuringExecution:

manager/manifests/istio.yaml.j2

@@ -25,8 +25,8 @@ spec:
  k8s:
  resources:
  requests:
- cpu: 200m # default is 500m
- memory: 1.75Gi # default is 2048Mi == 2Gi
+ cpu: 100m # default is 500m
+ memory: 200Mi # default is 2048Mi == 2Gi
  cni:
  enabled: false
  ingressGateways:
@@ -74,7 +74,7 @@ spec:
  cpu: 100m
  memory: 128Mi
  limits:
- cpu: 2000m
+ cpu: 1000m
  memory: 1024Mi
  replicaCount: 1
  hpaSpec:
@@ -132,20 +132,23 @@ spec:
  targetPort: 15443
  resources:
  requests:
- cpu: 200m
+ cpu: 300m
  memory: 128Mi
  limits:
- cpu: 2000m
+ cpu: 1500m
  memory: 1024Mi
  replicaCount: 1
  hpaSpec:
  minReplicas: 1
- maxReplicas: 1 # edit autoscaleEnabled in values if increasing this
+ maxReplicas: 100 # edit autoscaleEnabled in values if increasing this
  metrics:
  - type: Resource
  resource:
  name: cpu
- targetAverageUtilization: 80
+ targetAverageUtilization: 90
+ resource:
+ name: mem
+ targetAverageUtilization: 90
  scaleTargetRef:
  apiVersion: apps/v1
  kind: Deployment

manager/manifests/operator.yaml.j2

@@ -58,10 +58,10 @@ spec:
  imagePullPolicy: Always
  resources:
  requests:
- cpu: 200m
+ cpu: 100m
  memory: 128Mi
  limits:
- cpu: 2000m
+ cpu: 1500m
  memory: 1024Mi
  ports:
  - containerPort: 8888

manager/manifests/prometheus-kube-state-metrics.yaml

@@ -234,6 +234,12 @@ spec:
  port: 8080
  initialDelaySeconds: 5
  timeoutSeconds: 5
+ nodeSelector:
+ prometheus: "true"
+ tolerations:
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor

manager/manifests/prometheus-monitoring.yaml.j2

@@ -30,6 +30,12 @@ metadata:
 spec:
  image: {{ config['image_prometheus'] }}
  serviceAccountName: prometheus
+ nodeSelector:
+ prometheus: "true"
+ tolerations:
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
  podMonitorSelector:
  matchExpressions:
  - key: "monitoring.cortex.dev"

manager/manifests/prometheus-node-exporter.yaml

@@ -153,12 +153,15 @@ spec:
  hostPID: true
  nodeSelector:
  kubernetes.io/os: linux
+ prometheus: "true"
  securityContext:
  runAsNonRoot: true
  runAsUser: 65534
  serviceAccountName: node-exporter
  tolerations:
- - operator: Exists
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
  volumes:
  - hostPath:
  path: /sys

manager/manifests/prometheus-operator.yaml

@@ -14199,6 +14199,11 @@ spec:
  allowPrivilegeEscalation: false
  nodeSelector:
  kubernetes.io/os: linux
+ prometheus: "true"
+ tolerations:
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
  securityContext:
  runAsNonRoot: true
  runAsUser: 65534

manager/manifests/prometheus-statsd-exporter.yaml

@@ -74,6 +74,12 @@ spec:
  volumeMounts:
  - name: statsd-mapping-config
  mountPath: /etc/prometheus-statsd-exporter
+ nodeSelector:
+ prometheus: "true"
+ tolerations:
+ - key: prometheus
+ operator: Exists
+ effect: NoSchedule
  volumes:
  - name: statsd-mapping-config
  configMap:

pkg/crds/config/manager/manager.yaml

@@ -46,11 +46,11 @@ spec:
  periodSeconds: 10
  resources:
  limits:
- cpu: 100m
- memory: 30Mi
+ cpu: 200m
+ memory: 100Mi
  requests:
- cpu: 100m
- memory: 20Mi
+ cpu: 200m
+ memory: 80Mi
  volumeMounts:
  - mountPath: /mnt/cluster.yaml
  name: cluster-config