Skip to content

Commit c9929f8

Browse files
vulkoingimroboquat
vulkoingim
authored and
roboquat
committed
Terraform resources for Harvester VM and LBs
1 parent 4d694d0 commit c9929f8

File tree

8 files changed

+386
-1
lines changed

8 files changed

+386
-1
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,11 @@ repos:
5252
language: system
5353
pass_filenames: false
5454

55+
- repo: https://github.com/antonbabenko/pre-commit-terraform
56+
rev: v1.75.0
57+
hooks:
58+
- id: terraform_fmt
59+
args:
60+
- --args=-diff
61+
5562
exclude: ^install/installer/.*/.*\.golden$

‎dev/preview/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
*.tfstate
44
*.tfstate.backup
55
*.plan
6+
*.tfvars

‎dev/preview/infrastructure/harvester/cloudinit.yaml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#cloud-config
2+
users:
3+
- name: ubuntu
4+
sudo: "ALL=(ALL) NOPASSWD: ALL"
5+
ssh_authorized_keys:
6+
- ${ssh_authorized_keys}
7+
chpasswd:
8+
list: |
9+
ubuntu:ubuntu
10+
expire: False
11+
write_files:
12+
- path: /etc/disable-services.sh
13+
permissions: '0755'
14+
content: |
15+
#!/bin/bash
16+
systemctl disable google-guest-agent &
17+
systemctl disable google-startup-scripts &
18+
systemctl disable google-osconfig-agent &
19+
systemctl disable google-oslogin-cache.timer &
20+
systemctl disable google-shutdown-scripts &
21+
systemctl stop google-guest-agent &
22+
systemctl stop google-startup-scripts &
23+
systemctl stop google-osconfig-agent &
24+
systemctl stop google-oslogin-cache.timer &
25+
systemctl stop google-shutdown-scripts &
26+
- path: /etc/ssh/sshd_config.d/101-change-ssh-port.conf
27+
permissions: '0644'
28+
owner: root
29+
content: 'Port 2200'
30+
31+
- path: /usr/local/bin/bootstrap-k3s.sh
32+
permissions: '0744'
33+
owner: root
34+
content: |
35+
#!/bin/bash
36+
37+
set -eo pipefail
38+
39+
cat <<EOF >> /etc/containerd/config.toml
40+
[plugins."io.containerd.grpc.v1.cri".registry.configs."registry-1.docker.io".auth]
41+
username = "${dockerhub_user}"
42+
password = "${dockerhub_passwd}"
43+
EOF
44+
sudo systemctl restart containerd.service
45+
46+
# inspired by https://github.com/gitpod-io/ops/blob/main/deploy/workspace/templates/bootstrap.sh
47+
48+
# Install k3s
49+
export INSTALL_K3S_SKIP_DOWNLOAD=true
50+
51+
/usr/local/bin/install-k3s.sh \
52+
--token "1234" \
53+
--node-ip "$(hostname -I | cut -d ' ' -f1)" \
54+
--node-label "cloud.google.com/gke-nodepool=control-plane-pool" \
55+
--container-runtime-endpoint=/var/run/containerd/containerd.sock \
56+
--write-kubeconfig-mode 444 \
57+
--disable traefik \
58+
--disable metrics-server \
59+
--flannel-backend=none \
60+
--kubelet-arg config=/etc/kubernetes/kubelet-config.json \
61+
--kubelet-arg cgroup-driver=systemd \
62+
--kubelet-arg feature-gates=LocalStorageCapacityIsolation=true \
63+
--kubelet-arg feature-gates=LocalStorageCapacityIsolationFSQuotaMonitoring=true \
64+
--kube-apiserver-arg feature-gates=LocalStorageCapacityIsolation=true \
65+
--kube-apiserver-arg feature-gates=LocalStorageCapacityIsolationFSQuotaMonitoring=true \
66+
--cluster-init
67+
68+
# Seems like this is a bit flaky now, with k3s not always being ready, and the labeling
69+
# failing occasionally. Sleeping for a bit solves it.
70+
sleep 10
71+
72+
kubectl label nodes ${vm_name} \
73+
gitpod.io/workload_meta=true \
74+
gitpod.io/workload_ide=true \
75+
gitpod.io/workload_workspace_services=true \
76+
gitpod.io/workload_workspace_regular=true \
77+
gitpod.io/workload_workspace_headless=true \
78+
gitpod.io/workspace_0=true \
79+
gitpod.io/workspace_1=true \
80+
gitpod.io/workspace_2=true
81+
82+
# apply fix from https://github.com/k3s-io/klipper-lb/issues/6 so we can use the klipper servicelb
83+
# this can be removed if https://github.com/gitpod-io/gitpod-packer-gcp-image/pull/20 gets merged
84+
cat /var/lib/gitpod/manifests/calico.yaml | sed s/__KUBERNETES_NODE_NAME__\"\,/__KUBERNETES_NODE_NAME__\",\ \"container_settings\"\:\ \{\ \"allow_ip_forwarding\"\:\ true\ \}\,/ > /var/lib/gitpod/manifests/calico2.yaml
85+
86+
sed -i 's/docker.io/quay.io/g' /var/lib/gitpod/manifests/calico2.yaml
87+
sed -i 's/interface=ens/interface=en/g' /var/lib/gitpod/manifests/calico2.yaml
88+
sed -i 's/\$CLUSTER_IP_RANGE/10.20.0.0\/16/g' /var/lib/gitpod/manifests/calico2.yaml
89+
90+
kubectl apply -f /var/lib/gitpod/manifests/calico2.yaml
91+
92+
kubectl apply -f /var/lib/gitpod/manifests/cert-manager.yaml
93+
kubectl apply -f /var/lib/gitpod/manifests/metrics-server.yaml
94+
95+
# install CSI snapshotter CRDs and snapshot controller
96+
kubectl apply -f /var/lib/gitpod/manifests/csi-driver.yaml || true
97+
kubectl apply -f /var/lib/gitpod/manifests/csi-config.yaml || true
98+
99+
cat <<EOF >> /etc/bash.bashrc
100+
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
101+
EOF
102+
runcmd:
103+
- bash /etc/disable-services.sh
104+
- bash /usr/local/bin/bootstrap-k3s.sh

‎dev/preview/infrastructure/harvester/lb.tf‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
resource "kubernetes_deployment" "dev-loadbalancer" {
2+
provider = k8s.dev
3+
4+
metadata {
5+
name = "lb-${var.preview_name}"
6+
namespace = "loadbalancers"
7+
labels = {
8+
"gitpod.io/lbName" = var.preview_name
9+
}
10+
}
11+
12+
spec {
13+
replicas = 1
14+
15+
selector {
16+
match_labels = {
17+
"gitpod.io/lbName" = var.preview_name
18+
}
19+
}
20+
21+
template {
22+
metadata {
23+
name = "lb"
24+
labels = {
25+
"gitpod.io/lbName" = var.preview_name
26+
}
27+
}
28+
29+
spec {
30+
service_account_name = "proxy"
31+
enable_service_links = false
32+
33+
volume {
34+
name = "kubeconfig"
35+
secret {
36+
secret_name = "harvester-kubeconfig"
37+
}
38+
}
39+
40+
container {
41+
image = "bitnami/kubectl:1.25.2"
42+
name = "kubectl"
43+
args = [
44+
"port-forward",
45+
"--kubeconfig",
46+
"/mnt/kubeconfig/harvester-kubeconfig.yml",
47+
"-n",
48+
kubernetes_namespace.preview_namespace.metadata[0].name,
49+
"--address=0.0.0.0",
50+
"--pod-running-timeout=2m",
51+
"svc/proxy",
52+
"4430:443",
53+
"2200:22",
54+
]
55+
56+
volume_mount {
57+
mount_path = "/mnt/kubeconfig/"
58+
name = "kubeconfig"
59+
read_only = true
60+
}
61+
}
62+
}
63+
}
64+
}
65+
}

‎dev/preview/infrastructure/harvester/namespace.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
resource "kubernetes_namespace" "example" {
1+
resource "kubernetes_namespace" "preview_namespace" {
22
provider = k8s.harvester
33
metadata {
44
name = "preview-${var.preview_name}"

‎dev/preview/infrastructure/harvester/svc.tf‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Load balancer in the DEV cluster
2+
resource "kubernetes_service" "dev-svc" {
3+
provider = k8s.dev
4+
metadata {
5+
name = "lb-${var.preview_name}"
6+
namespace = "loadbalancers"
7+
}
8+
spec {
9+
port {
10+
name = "ssh-gateway"
11+
protocol = "TCP"
12+
port = 22
13+
target_port = 2200
14+
}
15+
port {
16+
name = "https"
17+
protocol = "TCP"
18+
port = 443
19+
target_port = 4430
20+
}
21+
selector = {
22+
"gitpod.io/lbName" = var.preview_name
23+
}
24+
type = "LoadBalancer"
25+
}
26+
}
27+
28+
# Proxy service in the HARVESTER cluster, same namespace
29+
resource "kubernetes_service" "harvester-svc" {
30+
provider = k8s.harvester
31+
metadata {
32+
name = "proxy"
33+
namespace = kubernetes_namespace.preview_namespace.metadata[0].name
34+
}
35+
36+
spec {
37+
port {
38+
name = "ssh-gateway"
39+
protocol = "TCP"
40+
port = 22
41+
target_port = 22
42+
}
43+
port {
44+
name = "vm-ssh"
45+
protocol = "TCP"
46+
port = 2200
47+
target_port = 2200
48+
}
49+
port {
50+
name = "http"
51+
protocol = "TCP"
52+
port = 80
53+
target_port = 80
54+
}
55+
port {
56+
name = "https"
57+
protocol = "TCP"
58+
port = 443
59+
target_port = 4430
60+
}
61+
port {
62+
name = "kube-api"
63+
protocol = "TCP"
64+
port = 6443
65+
target_port = 6443
66+
}
67+
port {
68+
name = "prometheus"
69+
protocol = "TCP"
70+
port = 9090
71+
target_port = 32001
72+
}
73+
port {
74+
name = "grafana"
75+
protocol = "TCP"
76+
port = 3000
77+
target_port = 32000
78+
}
79+
80+
selector = {
81+
"harvesterhci.io/vmName" = var.preview_name
82+
}
83+
84+
type = "ClusterIP"
85+
}
86+
}

‎dev/preview/infrastructure/harvester/variables.tf‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,37 @@ variable "dev_kube_path" {
1414
description = "The path to the Dev Cluster kubeconfig"
1515
default = "~/.kube/dev"
1616
}
17+
18+
variable "vm_memory" {
19+
type = string
20+
default = "2Gi"
21+
description = "Memory for the VM"
22+
}
23+
24+
variable "vm_cpu" {
25+
type = number
26+
default = 2
27+
description = "CPU for the VM"
28+
}
29+
30+
variable "vm_image" {
31+
type = string
32+
default = "default/gitpod-k3s-202209251218"
33+
description = "The image name"
34+
}
35+
36+
variable "dockerhub_user" {
37+
type = string
38+
description = "The dockerhub user used to pull images in k3s"
39+
}
40+
41+
variable "dockerhub_password" {
42+
type = string
43+
description = "The password for the dockerhub user used to pull images in k3s"
44+
}
45+
46+
variable "vm_storage_class" {
47+
type = string
48+
default = "longhorn-gitpod-k3s-202209251218-onereplica"
49+
description = "The storage class for the VM"
50+
}

0 commit comments

Comments
 (0)