[installer]: output all images to allow easy mirroring of containers

The output gives the original image name and tag and generates the new image name to be used

Author: Simon Emms

installer/cmd/mirror.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package cmd
+
+import (
+"github.com/spf13/cobra"
+)
+
+// validateCmd represents the validate command
+var mirrorCmd = &cobra.Command{
+Use: "mirror",
+Short: "Performs mirroring tasks",
+}
+
+func init() {
+rootCmd.AddCommand(mirrorCmd)
+}

installer/cmd/mirror_list.go

@@ -0,0 +1,197 @@
+// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package cmd
+
+import (
+"encoding/json"
+"fmt"
+"os"
+"regexp"
+"sort"
+"strings"
+
+"github.com/docker/distribution/reference"
+"github.com/gitpod-io/gitpod/installer/pkg/common"
+"github.com/spf13/cobra"
+)
+
+type mirrorListRepo struct {
+Original string `json:"original"`
+Target string `json:"target"`
+}
+
+var mirrorListOpts struct {
+ConfigFN string
+ExcludeThirdParty bool
+}
+
+// mirrorListCmd represents the mirror list command
+var mirrorListCmd = &cobra.Command{
+Use: "list",
+Short: "Renders a list of images used so they can be mirrored to a third-party registry",
+Long: `Renders a list of images used so they can be mirrored to a third-party registry
+
+A config file is required which can be generated with the init command.
+The "repository" field must be set to your container repository server
+address and this value will be used to generate the mirrored image names
+and tags.
+
+The output can then be used to iterate over each image. A script can
+be written to pull from the "original" path and then tag and push the
+image to the "target" repo`,
+Example: `
+ gitpod-installer mirror list --config config.yaml > mirror.json
+
+ # Pull original and push to target
+ for row in $(gitpod-installer mirror list --config ./config.yaml | jq -c '.[]'); do
+ original=$(echo $row | jq -r '.original')
+ target=$(echo $row | jq -r '.target')
+ docker pull $original
+ docker tag $original $target
+ docker push $target
+ done`,
+RunE: func(cmd *cobra.Command, args []string) error {
+if mirrorListOpts.ConfigFN == "" {
+return fmt.Errorf("config is a required flag")
+}
+
+_, cfgVersion, cfg, err := loadConfig(mirrorListOpts.ConfigFN)
+if err != nil {
+return err
+}
+
+// Throw error if set to the default Gitpod repository
+if cfg.Repository == common.GitpodContainerRegistry {
+return fmt.Errorf("cannot mirror images to repository %s", common.GitpodContainerRegistry)
+}
+
+// Get the target repository from the config
+targetRepo := strings.TrimRight(cfg.Repository, "/")
+
+// Use the default Gitpod registry to pull from
+cfg.Repository = common.GitpodContainerRegistry
+
+k8s, err := renderKubernetesObjects(cfgVersion, cfg)
+if err != nil {
+return err
+}
+
+// Map of images used for deduping
+allImages := make(map[string]bool)
+
+rawImages := make([]string, 0)
+for _, item := range k8s {
+rawImages = append(rawImages, getPodImages(item)...)
+rawImages = append(rawImages, getGenericImages(item)...)
+}
+
+images := make([]mirrorListRepo, 0)
+for _, img := range rawImages {
+// Dedupe
+if _, ok := allImages[img]; ok {
+continue
+}
+allImages[img] = true
+
+// Convert target
+target := img
+if strings.Contains(img, cfg.Repository) {
+// This is the Gitpod registry
+target = strings.Replace(target, cfg.Repository, targetRepo, 1)
+} else if !mirrorListOpts.ExcludeThirdParty {
+// Amend third-party images - remove the first part
+thirdPartyImg := strings.Join(strings.Split(img, "/")[1:], "/")
+target = fmt.Sprintf("%s/%s", targetRepo, thirdPartyImg)
+} else {
+// Excluding third-party images - just skip this one
+continue
+}
+
+images = append(images, mirrorListRepo{
+Original: img,
+Target: target,
+})
+}
+
+// Sort it by the Original
+sort.Slice(images, func(i, j int) bool {
+scoreI := images[i].Original
+scoreJ := images[j].Original
+
+return scoreI < scoreJ
+})
+
+fc, err := json.MarshalIndent(images, "", " ")
+if err != nil {
+return err
+}
+
+fmt.Println(string(fc))
+
+return nil
+},
+}
+
+func init() {
+mirrorCmd.AddCommand(mirrorListCmd)
+
+mirrorListCmd.Flags().BoolVar(&mirrorListOpts.ExcludeThirdParty, "exclude-third-party", false, "exclude non-Gitpod images")
+mirrorListCmd.Flags().StringVarP(&mirrorListOpts.ConfigFN, "config", "c", os.Getenv("GITPOD_INSTALLER_CONFIG"), "path to the config file")
+}
+
+// getGenericImages this is a bit brute force - anything starting "docker.io" or with Gitpod repo is found
+// this will be in ConfigMaps and could be anything, so will need cleaning up
+func getGenericImages(k8sObj string) []string {
+var images []string
+
+// Search for anything that matches docker.io or the Gitpod repo - docker.io needed for gitpod/workspace-full
+re := regexp.MustCompile(fmt.Sprintf("%s(.*)|%s(.*)", "docker.io", common.GitpodContainerRegistry))
+img := re.FindAllString(k8sObj, -1)
+
+if len(img) > 0 {
+for _, i := range img {
+// Remove whitespace
+i = strings.TrimSpace(i)
+// Remove end commas
+i = strings.TrimRight(i, ",")
+// Remove wrapping quotes
+i = strings.Trim(i, "\"")
+// Validate the image - assumes images are already fully qualified names
+_, err := reference.ParseNamed(i)
+if err != nil {
+// Invalid - ignore
+continue
+}
+
+images = append(images, i)
+}
+}
+
+return images
+}
+
+// getPodImages these are images that are found in the "image:" tag in a PodSpec
+// may be multiple tags in a file
+func getPodImages(k8sObj string) []string {
+var images []string
+
+re := regexp.MustCompile("image:(.*)")
+img := re.FindAllString(k8sObj, -1)
+
+if len(img) > 0 {
+for _, i := range img {
+// Remove "image":
+i = re.ReplaceAllString(i, "$1")
+// Remove whitespace
+i = strings.TrimSpace(i)
+// Remove wrapping quotes
+i = strings.Trim(i, "\"")
+
+images = append(images, i)
+}
+}
+
+return images
+}