Move rollout and analysis into separate packages

This also includes: - make actions using RolloutAction Interface - abstract out the analysis logic into a separate package - bugfix: don't close the channel - working prototype with metric analysis - logs refactor Signed-off-by: Tarun Pothulapati <tarun@gitpod.io>

Author: Pothulapati

components/workspace-rollout-job/cmd/root.go

@@ -5,10 +5,13 @@
 package cmd
 
 import (
+"context"
 "fmt"
 "os"
+"time"
 
 "github.com/gitpod-io/gitpod/common-go/log"
+"github.com/gitpod-io/gitpod/workspace-rollout-job/pkg/analysis"
 "github.com/gitpod-io/gitpod/workspace-rollout-job/pkg/rollout"
 "github.com/gitpod-io/gitpod/workspace-rollout-job/pkg/wsbridge"
 "github.com/spf13/cobra"
@@ -19,6 +22,7 @@ var rootCmd = &cobra.Command{
 Short: "Rollout from old to a new cluster while monitoring metrics",
 Run: func(cmd *cobra.Command, args []string) {
 log.Info("Starting workspace-rollout-job")
+ctx := context.Background()
 var err error
 old, presence := os.LookupEnv("OLD_CLUSTER")
 if !presence {
@@ -30,20 +34,22 @@ var rootCmd = &cobra.Command{
 log.WithError(err).Fatal("cannot get new cluster")
 }
 
+wsManagerBridgeClient := wsbridge.NewWsManagerBridgeClient("localhost:8080")
 // Check if the old cluster has a 100 score.
-if err = wsbridge.CheckScore(old, 100); err != nil {
+if score, err := wsManagerBridgeClient.GetScore(ctx, old); err != nil || score != 100 {
 log.WithError(err).Fatal("init condition does not satisfy")
 }
 
 // Check if the new cluster has a 0 zero score.
 // TODO: Check if the new cluster has no constraints.
-if err = wsbridge.CheckScore(new, 0); err != nil {
+if score, err := wsManagerBridgeClient.GetScore(ctx, new); err != nil || score != 0 {
 log.WithError(err).Fatal("init condition does not satisfy")
 }
 
-// Start the rollout process.
-job := rollout.New(old, new, "http://prometheus:9090")
-job.Start()
+// Start the rollout process
+prometheusAnalyzer := analysis.NewPrometheusAnalyzer("http://localhost:9090")
+job := rollout.New(old, new, 20*time.Second, 1*time.Second, 10, prometheusAnalyzer, wsManagerBridgeClient)
+job.Start(ctx)
 },
 }
 

components/workspace-rollout-job/go.mod

@@ -6,24 +6,28 @@ require (
 github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 github.com/gitpod-io/gitpod/ws-manager-bridge/api v0.0.0-00010101000000-000000000000
 github.com/prometheus/client_golang v1.13.0
+github.com/prometheus/common v0.37.0
 github.com/spf13/cobra v1.4.0
+github.com/stretchr/testify v1.7.0
 google.golang.org/grpc v1.49.0
 )
 
 require (
+github.com/davecgh/go-spew v1.1.1 // indirect
 github.com/golang/protobuf v1.5.2 // indirect
 github.com/inconshreveable/mousetrap v1.0.0 // indirect
 github.com/json-iterator/go v1.1.12 // indirect
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 github.com/modern-go/reflect2 v1.0.2 // indirect
-github.com/prometheus/common v0.37.0 // indirect
+github.com/pmezard/go-difflib v1.0.0 // indirect
 github.com/sirupsen/logrus v1.8.1 // indirect
 github.com/spf13/pflag v1.0.5 // indirect
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect
 golang.org/x/text v0.3.7 // indirect
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
 google.golang.org/protobuf v1.28.1 // indirect
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
 replace github.com/gitpod-io/gitpod/ws-manager-bridge/api => ../ws-manager-bridge-api/go // leeway

components/workspace-rollout-job/go.sum

@@ -0,0 +1,15 @@
+// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package analysis
+
+import (
+"context"
+)
+
+type Analyzer interface {
+// Given a cluster name, MoveForward is called by the rollout routine
+// repeatedly to determine whether to move forward on the rollout or not.
+MoveForward(ctx context.Context, clusterName string) (bool, error)
+}

components/workspace-rollout-job/pkg/analysis/analyzer.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package analysis
+
+import (
+"context"
+"fmt"
+"math"
+"time"
+
+logrus "github.com/gitpod-io/gitpod/common-go/log"
+"github.com/prometheus/client_golang/api"
+v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+"github.com/prometheus/common/model"
+)
+
+const (
+// Rate of increase in error count
+errorMetric = "sum by (cluster) (rate(gitpod_ws_manager_workspace_starts_failure_total{cluster=~\"%s.*\"}[%dms]))"
+)
+
+type PrometheusAnalyzer struct {
+prometheusURL string
+startTime time.Time
+}
+
+func NewPrometheusAnalyzer(promURL string) *PrometheusAnalyzer {
+return &PrometheusAnalyzer{
+prometheusURL: promURL,
+startTime: time.Now(),
+}
+}
+
+func (pa *PrometheusAnalyzer) MoveForward(ctx context.Context, clusterName string) (bool, error) {
+log := logrus.WithField("component", "prometheus-analyzer")
+client, err := api.NewClient(api.Config{
+Address: pa.prometheusURL,
+})
+if err != nil {
+return false, err
+}
+
+v1api := v1.NewAPI(client)
+ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+defer cancel()
+
+queryResult, warnings, err := v1api.Query(ctx, fmt.Sprintf(errorMetric, clusterName, time.Since(pa.startTime).Milliseconds()), time.Now())
+if err != nil {
+return false, err
+}
+if len(warnings) > 0 {
+log.Warnf("Warnings: %v\n", warnings)
+}
+
+result, ok := queryResult.(model.Vector)
+if !ok {
+return false, fmt.Errorf("unexpected result type: %T", queryResult)
+}
+
+if len(result) != 1 {
+if len(result) == 0 {
+log.Infof("No error data found for %s. Proceeding", clusterName)
+return true, nil
+}
+return false, fmt.Errorf("unexpected result Prometheus result vector length: %d", len(result))
+}
+val := float64(result[0].Value)
+if math.IsNaN(val) {
+return false, fmt.Errorf("unexpected sample value: %v", result[0].Value)
+}
+
+// Return true if the error rate is 0
+if val > 0 {
+log.Infof("Found error metric rate as %f ", val)
+return false, nil
+}
+
+return true, nil
+}