[werft] Integrate Installer with Deploy

The Installer is the default for new Preview Environments. Preview Environments powered by Helm can switch to the Installer if 'with-clean-slate-deployment=true' is specified. Folks can continue using Helm if 'with-helm=true' is specified. Co-authored-by: Christian Weichel <chris@gitpod.io>

Author: 2 people

.werft/build.ts

@@ -5,29 +5,53 @@ import { getGlobalWerftInstance } from './werft';
 
 export const IS_PREVIEW_APP_LABEL: string = "isPreviewApp";
 
+export const helmInstallName = "gitpod";
+
 export function setKubectlContextNamespace(namespace: string, shellOpts: ExecOptions) {
  [
  `kubectl config current-context`,
  `kubectl config set-context --current --namespace=${namespace}`
  ].forEach(cmd => exec(cmd, shellOpts));
 }
 
+export async function wipePreviewEnvironmentAndNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+ // wipe preview envs built with installer
+ await wipePreviewEnvironmentInstaller(namespace, shellOpts);
+
+ // wipe preview envs previously built with helm
+ await wipePreviewEnvironmentHelm(helmInstallName, namespace, shellOpts)
+
+ deleteAllWorkspaces(namespace, shellOpts);
+
+ await deleteAllUnnamespacedObjects(namespace, shellOpts);
+
+ deleteNamespace(true, namespace, shellOpts);
+}
+
 export async function wipeAndRecreateNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
- await wipePreviewEnvironment(helmInstallName, namespace, shellOpts);
+ await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, shellOpts);
 
  createNamespace(namespace, shellOpts);
 }
 
-export async function wipePreviewEnvironment(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+export async function wipePreviewEnvironmentHelm(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
  // uninstall helm first so that:
  // - ws-scaler can't create new ghosts in the meantime
  // - ws-manager can't start new probes/workspaces
  uninstallHelm(helmInstallName, namespace, shellOpts)
+}
 
- deleteAllWorkspaces(namespace, shellOpts);
- await deleteAllUnnamespacedObjects(namespace, shellOpts);
+async function wipePreviewEnvironmentInstaller(namespace: string, shellOpts: ExecOptions) {
+ const slice = shellOpts.slice || "installer";
+ const werft = getGlobalWerftInstance();
 
- deleteNamespace(true, namespace, shellOpts);
+ const hasGitpodConfigmap = (exec(`kubectl -n ${namespace} get configmap gitpod-app`, { slice, dontCheckRc: true })).code === 0;
+ if (hasGitpodConfigmap) {
+ werft.log(slice, `${namespace} has Gitpod configmap, proceeding with removal`);
+ exec(`./.werft/util/uninstall-gitpod.sh ${namespace}`, { slice });
+ } else {
+ werft.log(slice, `There is no Gitpod configmap, moving on`);
+ }
 }
 
 function uninstallHelm(installationName: string, namespace: string, shellOpts: ExecOptions) {
@@ -43,6 +67,7 @@ function uninstallHelm(installationName: string, namespace: string, shellOpts: E
  exec(`helm --namespace ${namespace} delete ${installationName} --wait`, shellOpts);
 }
 
+// Delete pods for running workspaces, even if they are stuck in terminating because of the finalizer decorator
 function deleteAllWorkspaces(namespace: string, shellOpts: ExecOptions) {
  const objs = exec(`kubectl get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, async: false })
  .split("\n")
@@ -74,13 +99,14 @@ async function deleteAllUnnamespacedObjects(namespace: string, shellOpts: ExecOp
 
  const promisedDeletes: Promise<any>[] = [];
  for (const resType of ["clusterrole", "clusterrolebinding", "podsecuritypolicy"]) {
- werft.log(slice, `Deleting old ${resType}s...`);
+ werft.log(slice, `Searching and filtering ${resType}s...`);
  const objs = exec(`kubectl get ${resType} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, slice, async: false })
  .split("\n")
  .map(o => o.trim())
  .filter(o => o.length > 0)
  .filter(o => o.startsWith(`${namespace}-ns-`)); // "{{ .Release.Namespace }}-ns-" is the prefix-pattern we use throughout our helm resources for un-namespaced resources
 
+ werft.log(slice, `Deleting old ${resType}s...`);
  for (const obj of objs) {
  promisedDeletes.push(exec(`kubectl delete ${resType} ${obj}`, { ...shellOpts, slice, async: true }) as Promise<any>);
  }
@@ -127,7 +153,7 @@ export function deleteNamespace(wait: boolean, namespace: string, shellOpts: Exe
  }
 }
 
-export function deleteNonNamespaceObjects(namespace: string, destname: string, shellOpts: ExecOptions) {
+export async function deleteNonNamespaceObjects(namespace: string, destname: string, shellOpts: ExecOptions) {
  exec(`/usr/local/bin/helm3 delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, { ...shellOpts });
 
  let objs = [];
@@ -141,9 +167,11 @@ export function deleteNonNamespaceObjects(namespace: string, destname: string, s
  )
  )
 
+ const promisedDeletes: Promise<any>[] = [];
  objs.forEach(o => {
- exec(`kubectl delete ${o.kind} ${o.obj}`, shellOpts);
+ promisedDeletes.push(exec(`kubectl delete ${o.kind} ${o.obj}`, {...shellOpts, async: true}) as Promise<any>);
  });
+ await Promise.all(promisedDeletes);
 }
 
 export interface PortRange {

.werft/post-process.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+NAMESPACE=$1
+
+if [[ -z ${NAMESPACE} ]]; then
+ echo "One or more input params were invalid. The params we received were: ${NAMESPACE}"
+ exit 1
+fi
+
+echo "Removing Gitpod in namespace ${NAMESPACE}"
+kubectl get configmap gitpod-app -n "${NAMESPACE}" -o jsonpath='{.data.app\.yaml}' | kubectl delete -f -
+
+echo "Removing Gitpod storage from ${NAMESPACE}"
+kubectl -n "${NAMESPACE}" delete pvc data-mysql-0
+# the installer includes the minio PVC in it's config mpap, this is a "just in case"
+kubectl -n "${NAMESPACE}" delete pvc minio || true
+
+echo "Successfully removed Gitpod from ${NAMESPACE}"

.werft/util/kubectl.ts

@@ -1,5 +1,5 @@
 import { Werft } from './util/werft'
-import { wipePreviewEnvironment, listAllPreviewNamespaces } from './util/kubectl';
+import { wipePreviewEnvironmentAndNamespace, listAllPreviewNamespaces, helmInstallName } from './util/kubectl';
 import * as fs from 'fs';
 import { deleteExternalIp } from './util/gcloud';
 import * as Tracing from './observability/tracing'
@@ -23,7 +23,7 @@ async function wipePreviewCluster(shellOpts: ExecOptions) {
  }
 
  for (const namespace of namespaces) {
- await wipePreviewEnvironment("gitpod", namespace, { ...shellOpts, slice: 'wipe' });
+ await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, { ...shellOpts, slice: 'wipe' });
  }
 }
 

.werft/util/uninstall-gitpod.sh

@@ -6,6 +6,7 @@ package cluster
 
 import (
 "fmt"
+
 "github.com/gitpod-io/gitpod/installer/pkg/common"
 v1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
 cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
@@ -22,8 +23,9 @@ func certmanager(ctx *common.RenderContext) ([]runtime.Object, error) {
 &v1.Issuer{
 TypeMeta: common.TypeMetaCertificateIssuer,
 ObjectMeta: metav1.ObjectMeta{
-Name: caIssuer,
-Labels: common.DefaultLabels(Component),
+Name: caIssuer,
+Labels: common.DefaultLabels(Component),
+Namespace: ctx.Namespace,
 },
 Spec: v1.IssuerSpec{IssuerConfig: v1.IssuerConfig{
 SelfSigned: &v1.SelfSignedIssuer{},

.werft/wipe-devstaging.ts

@@ -18,9 +18,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 return []runtime.Object{&rbacv1.ClusterRole{
 TypeMeta: common.TypeMetaClusterRole,
 ObjectMeta: metav1.ObjectMeta{
-Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-Namespace: ctx.Namespace,
-Labels: common.DefaultLabels(Component),
+Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+Labels: common.DefaultLabels(Component),
 },
 Rules: []rbacv1.PolicyRule{{
 APIGroups: []string{"policy"},

installer/pkg/components/cluster/certmanager.go

@@ -19,9 +19,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 &rbacv1.ClusterRole{
 TypeMeta: common.TypeMetaClusterRole,
 ObjectMeta: metav1.ObjectMeta{
-Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-Namespace: ctx.Namespace,
-Labels: common.DefaultLabels(Component),
+Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+Labels: common.DefaultLabels(Component),
 },
 Rules: []rbacv1.PolicyRule{{
 APIGroups: []string{"policy"},

installer/pkg/components/image-builder-mk3/clusterrole.go

@@ -6,6 +6,7 @@ package registryfacade
 
 import (
 "fmt"
+
 "github.com/gitpod-io/gitpod/installer/pkg/common"
 
 "k8s.io/api/policy/v1beta1"
@@ -17,9 +18,8 @@ func podsecuritypolicy(ctx *common.RenderContext) ([]runtime.Object, error) {
 return []runtime.Object{&v1beta1.PodSecurityPolicy{
 TypeMeta: common.TypeMetaPodSecurityPolicy,
 ObjectMeta: metav1.ObjectMeta{
-Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-Namespace: ctx.Namespace,
-Labels: common.DefaultLabels(Component),
+Name: fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+Labels: common.DefaultLabels(Component),
 Annotations: map[string]string{
 "seccomp.security.alpha.kubernetes.io/allowedProfileNames": "runtime/default",
 "apparmor.security.beta.kubernetes.io/allowedProfileNames": "runtime/default",

installer/pkg/components/registry-facade/clusterrole.go

@@ -7,6 +7,7 @@ package server
 import (
 "encoding/json"
 "fmt"
+
 "github.com/gitpod-io/gitpod/installer/pkg/common"
 "github.com/gitpod-io/gitpod/installer/pkg/components/workspace"
 corev1 "k8s.io/api/core/v1"
@@ -54,7 +55,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 MinAgeDays: 14,
 MinAgePrebuildDays: 7,
 },
-EnableLocalApp: false,
+EnableLocalApp: true,
 AuthProviderConfigs: ctx.Config.AuthProviders,
 BuiltinAuthProvidersConfigured: len(ctx.Config.AuthProviders) > 0,
 DisableDynamicAuthProviderLogin: false,