ci: run integration test against mainline and stable versions (coder#248)

- Adds `scripts/coderversion` to fetch stable and mainline versions - Adds `minVersion` constraint to integration tests - Runs integration tests against both stable and mainline versions - Adds separate integration test for fields introduced in v2.12.0

Author: johnstcn

.github/workflows/test.yml

@@ -38,13 +38,21 @@ jobs:
  run: |
  go build -v .
 
- - name: Run integration test
+ - name: Run integration test (mainline)
  timeout-minutes: 10
  env:
  CODER_IMAGE: "ghcr.io/coder/coder"
- CODER_VERSION: "latest"
  run: |
- go test -v ./integration
+ source <(go run ./scripts/coderversion)
+ CODER_VERSION="${CODER_MAINLINE_VERSION}" go test -v ./integration
+
+ - name: Run integration test (stable)
+ timeout-minutes: 10
+ env:
+ CODER_IMAGE: "ghcr.io/coder/coder"
+ run: |
+ source <(go run ./scripts/coderversion)
+ CODER_VERSION="${CODER_STABLE_VERSION}" go test -v ./integration
 
  # run acceptance tests in a matrix with Terraform core versions
  test:

go.mod

@@ -9,13 +9,17 @@ require (
 github.com/google/uuid v1.6.0
 github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320
 github.com/hashicorp/terraform-plugin-sdk/v2 v2.20.0
+github.com/masterminds/semver v1.5.0
 github.com/mitchellh/mapstructure v1.5.0
 github.com/robfig/cron/v3 v3.0.1
 github.com/stretchr/testify v1.9.0
+golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
+golang.org/x/mod v0.18.0
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 )
 
 require (
+github.com/Masterminds/semver v1.5.0 // indirect
 github.com/Microsoft/go-winio v0.5.2 // indirect
 github.com/agext/levenshtein v1.2.3 // indirect
 github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
@@ -76,8 +80,6 @@ require (
 go.opentelemetry.io/otel/sdk v1.27.0 // indirect
 go.opentelemetry.io/otel/trace v1.27.0 // indirect
 golang.org/x/crypto v0.23.0 // indirect
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
-golang.org/x/mod v0.18.0 // indirect
 golang.org/x/net v0.25.0 // indirect
 golang.org/x/sys v0.20.0 // indirect
 golang.org/x/text v0.15.0 // indirect

go.sum

@@ -1,6 +1,8 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
@@ -147,6 +149,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/masterminds/semver v1.5.0 h1:hTxJTTY7tjvnWMrl08O6u3G6BLlKVwxSz01lVac9P8U=
+github.com/masterminds/semver v1.5.0/go.mod h1:s7KNT9fnd7edGzwwP7RBX4H0v/CYd5qdOLfkL1V75yg=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=

integration/integration_test.go

@@ -57,15 +57,19 @@ func TestIntegration(t *testing.T) {
 require.NoError(t, err, "invalid value specified for timeout")
 ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMins)*time.Minute)
 t.Cleanup(cancel)
+ctrID := setup(ctx, t, t.Name(), coderImg, coderVersion)
 
 for _, tt := range []struct {
 // Name of the folder under `integration/` containing a test template
 name string
+// Minimum coder version for which to run this test
+minVersion string
 // map of string to regex to be passed to assertOutput()
 expectedOutput map[string]string
 }{
 {
-name: "test-data-source",
+name: "test-data-source",
+minVersion: "v0.0.0",
 expectedOutput: map[string]string{
 "provisioner.arch": runtime.GOARCH,
 "provisioner.id": `[a-zA-Z0-9-]+`,
@@ -86,6 +90,31 @@ func TestIntegration(t *testing.T) {
 "workspace.template_name": `test-data-source`,
 "workspace.template_version": `.+`,
 "workspace.transition": `start`,
+},
+},
+{
+name: "workspace-owner",
+minVersion: "v2.12.0",
+expectedOutput: map[string]string{
+"provisioner.arch": runtime.GOARCH,
+"provisioner.id": `[a-zA-Z0-9-]+`,
+"provisioner.os": runtime.GOOS,
+"workspace.access_port": `\d+`,
+"workspace.access_url": `https?://\D+:\d+`,
+"workspace.id": `[a-zA-z0-9-]+`,
+"workspace.name": ``,
+"workspace.owner": `testing`,
+"workspace.owner_email": `testing@coder\.com`,
+"workspace.owner_groups": `\[\]`,
+"workspace.owner_id": `[a-zA-Z0-9]+`,
+"workspace.owner_name": `default`,
+"workspace.owner_oidc_access_token": `^$`, // TODO: need a test OIDC integration
+"workspace.owner_session_token": `[a-zA-Z0-9-]+`,
+"workspace.start_count": `1`,
+"workspace.template_id": `[a-zA-Z0-9-]+`,
+"workspace.template_name": `workspace-owner`,
+"workspace.template_version": `.+`,
+"workspace.transition": `start`,
 "workspace_owner.email": `testing@coder\.com`,
 "workspace_owner.full_name": `default`,
 "workspace_owner.groups": `\[\]`,
@@ -98,9 +127,13 @@ func TestIntegration(t *testing.T) {
 },
 },
 } {
+tt := tt
 t.Run(tt.name, func(t *testing.T) {
+t.Parallel()
+if coderVersion != "latest" && semver.Compare(coderVersion, tt.minVersion) < 0 {
+t.Skipf("skipping due to CODER_VERSION %q < minVersion %q", coderVersion, tt.minVersion)
+}
 // Given: we have an existing Coder deployment running locally
-ctrID := setup(ctx, t, tt.name, coderImg, coderVersion)
 // Import named template
 
 // NOTE: Template create command was deprecated after this version

integration/test-data-source/main.tf

@@ -38,15 +38,6 @@ locals {
  "workspace.template_name" : data.coder_workspace.me.template_name,
  "workspace.template_version" : data.coder_workspace.me.template_version,
  "workspace.transition" : data.coder_workspace.me.transition,
- "workspace_owner.email" : data.coder_workspace_owner.me.email,
- "workspace_owner.full_name" : data.coder_workspace_owner.me.full_name,
- "workspace_owner.groups" : jsonencode(data.coder_workspace_owner.me.groups),
- "workspace_owner.id" : data.coder_workspace_owner.me.id,
- "workspace_owner.name" : data.coder_workspace_owner.me.name,
- "workspace_owner.oidc_access_token" : data.coder_workspace_owner.me.oidc_access_token,
- "workspace_owner.session_token" : data.coder_workspace_owner.me.session_token,
- "workspace_owner.ssh_private_key" : data.coder_workspace_owner.me.ssh_private_key,
- "workspace_owner.ssh_public_key" : data.coder_workspace_owner.me.ssh_public_key,
  }
 }
 

integration/workspace-owner/main.tf

@@ -0,0 +1,65 @@
+terraform {
+ required_providers {
+ coder = {
+ source = "coder/coder"
+ }
+ local = {
+ source = "hashicorp/local"
+ }
+ }
+}
+
+// TODO: test coder_external_auth and coder_git_auth
+// data coder_external_auth "me" {}
+// data coder_git_auth "me" {}
+data "coder_provisioner" "me" {}
+data "coder_workspace" "me" {}
+data "coder_workspace_owner" "me" {}
+
+locals {
+ # NOTE: these must all be strings in the output
+ output = {
+ "provisioner.arch" : data.coder_provisioner.me.arch,
+ "provisioner.id" : data.coder_provisioner.me.id,
+ "provisioner.os" : data.coder_provisioner.me.os,
+ "workspace.access_port" : tostring(data.coder_workspace.me.access_port),
+ "workspace.access_url" : data.coder_workspace.me.access_url,
+ "workspace.id" : data.coder_workspace.me.id,
+ "workspace.name" : data.coder_workspace.me.name,
+ "workspace.owner" : data.coder_workspace.me.owner,
+ "workspace.owner_email" : data.coder_workspace.me.owner_email,
+ "workspace.owner_groups" : jsonencode(data.coder_workspace.me.owner_groups),
+ "workspace.owner_id" : data.coder_workspace.me.owner_id,
+ "workspace.owner_name" : data.coder_workspace.me.owner_name,
+ "workspace.owner_oidc_access_token" : data.coder_workspace.me.owner_oidc_access_token,
+ "workspace.owner_session_token" : data.coder_workspace.me.owner_session_token,
+ "workspace.start_count" : tostring(data.coder_workspace.me.start_count),
+ "workspace.template_id" : data.coder_workspace.me.template_id,
+ "workspace.template_name" : data.coder_workspace.me.template_name,
+ "workspace.template_version" : data.coder_workspace.me.template_version,
+ "workspace.transition" : data.coder_workspace.me.transition,
+ "workspace_owner.email" : data.coder_workspace_owner.me.email,
+ "workspace_owner.full_name" : data.coder_workspace_owner.me.full_name,
+ "workspace_owner.groups" : jsonencode(data.coder_workspace_owner.me.groups),
+ "workspace_owner.id" : data.coder_workspace_owner.me.id,
+ "workspace_owner.name" : data.coder_workspace_owner.me.name,
+ "workspace_owner.oidc_access_token" : data.coder_workspace_owner.me.oidc_access_token,
+ "workspace_owner.session_token" : data.coder_workspace_owner.me.session_token,
+ "workspace_owner.ssh_private_key" : data.coder_workspace_owner.me.ssh_private_key,
+ "workspace_owner.ssh_public_key" : data.coder_workspace_owner.me.ssh_public_key,
+ }
+}
+
+variable "output_path" {
+ type = string
+}
+
+resource "local_file" "output" {
+ filename = var.output_path
+ content = jsonencode(local.output)
+}
+
+output "output" {
+ value = local.output
+ sensitive = true
+}

scripts/coderversion/main.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+"encoding/json"
+"fmt"
+"net/http"
+"os"
+
+"github.com/masterminds/semver"
+)
+
+func main() {
+releases := fetchReleases()
+
+mainlineVer := semver.MustParse("v0.0.0")
+for _, rel := range releases {
+if rel == "" {
+debug("ignoring untagged version %s\n", rel)
+continue
+}
+
+ver, err := semver.NewVersion(rel)
+if err != nil {
+debug("skipping invalid version %s\n", rel)
+}
+
+if ver.Compare(mainlineVer) > 0 {
+mainlineVer = ver
+continue
+}
+}
+
+mainline := fmt.Sprintf("v%d.%d.%d", mainlineVer.Major(), mainlineVer.Minor(), mainlineVer.Patch())
+_, _ = fmt.Fprintf(os.Stdout, "CODER_MAINLINE_VERSION=%q\n", mainline)
+
+expectedStableMinor := mainlineVer.Minor() - 1
+if expectedStableMinor < 0 {
+expectedStableMinor = 0
+}
+debug("expected stable minor: %d\n", expectedStableMinor)
+stableVer := semver.MustParse("v0.0.0")
+for _, rel := range releases {
+debug("check version %s\n", rel)
+if rel == "" {
+debug("ignoring untagged version %s\n", rel)
+continue
+}
+
+ver, err := semver.NewVersion(rel)
+if err != nil {
+debug("skipping invalid version %s\n", rel)
+}
+
+if ver.Minor() != expectedStableMinor {
+debug("skipping version %s\n", rel)
+continue
+}
+
+if ver.Compare(stableVer) > 0 {
+stableVer = ver
+continue
+}
+}
+
+stable := fmt.Sprintf("v%d.%d.%d", stableVer.Major(), stableVer.Minor(), stableVer.Patch())
+_, _ = fmt.Fprintf(os.Stdout, "CODER_STABLE_VERSION=%q\n", stable)
+}
+
+type release struct {
+TagName string `json:"tag_name"`
+}
+
+const releasesURL = "https://api.github.com/repos/coder/coder/releases"
+
+// fetchReleases fetches the releases of coder/coder
+// this is done directly via JSON API to avoid pulling in the entire
+// github client
+func fetchReleases() []string {
+resp, err := http.Get(releasesURL)
+if err != nil {
+fatal("get releases: %w", err)
+}
+defer resp.Body.Close()
+
+var releases []release
+if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
+fatal("parse releases: %w", err)
+}
+
+var ss []string
+for _, rel := range releases {
+if rel.TagName != "" {
+ss = append(ss, rel.TagName)
+
+}
+}
+return ss
+}
+
+func debug(format string, args ...any) {
+if _, ok := os.LookupEnv("VERBOSE"); ok {
+_, _ = fmt.Fprintf(os.Stderr, format, args...)
+}
+}
+
+func fatal(format string, args ...any) {
+_, _ = fmt.Fprintf(os.Stderr, format, args...)
+os.Exit(1)
+}