feat(vet): Run rules against a managed database (#2751)

* feat(vet): Support managed databases * feat(vet): Run rules against a managed database * Add managed to JSON schema * Use different hostname for region discovery * docs: second pass at managed-databases.md --------- Co-authored-by: Andrew Benton <andrew@sqlc.dev>

Author: kyleconroy

.github/workflows/ci.yml

@@ -32,16 +32,6 @@ jobs:
  runs-on: ubuntu-latest
 
  services:
- postgres:
- image: "postgres:15"
- env:
- POSTGRES_DB: postgres
- POSTGRES_PASSWORD: postgres
- POSTGRES_USER: postgres
- ports:
- - 5432:5432
- # needed because the postgres container does not provide a healthcheck
- options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
  mysql:
  image: "mysql/mysql-server:8.0"
  env:
@@ -69,17 +59,13 @@ jobs:
  - name: test ./...
  run: gotestsum --junitfile junit.xml -- --tags=examples ./...
  env:
- PG_USER: postgres
- PG_HOST: localhost
- PG_DATABASE: postgres
- PG_PASSWORD: postgres
- PG_PORT: ${{ job.services.postgres.ports['5432'] }}
  MYSQL_DATABASE: mysql
  MYSQL_HOST: localhost
  MYSQL_PORT: ${{ job.services.mysql.ports['3306'] }}
  MYSQL_ROOT_PASSWORD: mysecretpassword
  CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }}
  CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
+ SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
 
  - name: build internal/endtoend
  run: go build ./...

docs/howto/managed-databases.md

@@ -0,0 +1,68 @@
+# Managed databases
+
+*Added in v1.22.0*
+
+`sqlc` can create and maintain hosted databases for your project. These
+databases are immediately useful for linting queries with [`sqlc vet`](vet.md)
+if your lint rules require a connection to a running database. PostgreSQL
+support is available today, with MySQL on the way.
+
+This feature is under active development, and we're interested in supporting
+other use-cases. Beyond linting queries, you can use sqlc managed databases
+in your tests to quickly stand up a database per test suite or even per test,
+providing a real, isolated database for a test run. No cleanup required.
+
+Interested in trying out managed databases? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email
+at <mailto:hello@sqlc.dev>.
+
+## Configuring managed databases
+
+To configure `sqlc` to use a managed database, remove the `uri` key from your
+`database` configuration and replace it with the `managed` key set to `true`.
+Set the `project` key in your `cloud` configuration to the value of your
+project ID, obtained via the sqlc.dev Dashboard.
+
+```yaml
+version: '2'
+cloud:
+ project: '<PROJECT_ID>'
+sql:
+- schema: schema.sql
+ queries: query.sql
+ engine: postgresql
+ database:
+ managed: true
+```
+
+## Authentication
+
+`sqlc` expects to find a valid auth token in the value of the `SQLC_AUTH_TOKEN`
+environment variable. You can create an auth token via the sqlc.dev Dashboard.
+
+```shell
+export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx
+```
+
+## Linting queries
+
+With managed databases configured, `sqlc vet` will create a database with your
+package's schema and use that database when running lint rules that require a
+database connection, e.g. any [rule relying on `EXPLAIN ...` output](vet.md#rules-using-explain-output).
+
+If you don't yet have any vet rules, the [built-in sqlc/db-prepare rule](vet.md#sqlc-db-prepare)
+is a good place to start. It prepares each of your queries against the database
+to ensure the query is valid. Here's a minimal working configuration:
+
+```yaml
+version: '2'
+cloud:
+ project: '<PROJECT_ID>'
+sql:
+- schema: schema.sql
+ queries: query.sql
+ engine: postgresql
+ database:
+ managed: true
+ rules:
+ - sqlc/db-prepare
+```

examples/authors/sqlc.yaml

@@ -1,10 +1,12 @@
 version: '2'
+cloud:
+ project: "01HAQMMECEYQYKFJN8MP16QC41"
 sql:
 - schema: postgresql/schema.sql
  queries: postgresql/query.sql
  engine: postgresql
  database:
- uri: postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/authors
+ managed: true
  rules:
  - sqlc/db-prepare
  - postgresql-query-too-costly

examples/batch/sqlc.json

@@ -1,5 +1,8 @@
 {
  "version": "1",
+ "cloud": {
+ "project": "01HAQMMECEYQYKFJN8MP16QC41"
+ },
  "packages": [
  {
  "path": "postgresql",
@@ -8,7 +11,7 @@
  "queries": "postgresql/query.sql",
  "engine": "postgresql",
  "database": {
- "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/batch"
+ "managed": true
  },
  "rules": [
  "sqlc/db-prepare"

examples/booktest/sqlc.json

@@ -1,5 +1,8 @@
 {
  "version": "1",
+ "cloud": {
+ "project": "01HAQMMECEYQYKFJN8MP16QC41"
+ },
  "packages": [
  {
  "name": "booktest",
@@ -8,7 +11,7 @@
  "queries": "postgresql/query.sql",
  "engine": "postgresql",
  "database": {
- "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/booktest"
+ "managed": true
  },
  "rules": [
  "sqlc/db-prepare"

examples/jets/sqlc.json

@@ -1,5 +1,8 @@
 {
  "version": "1",
+ "cloud": {
+ "project": "01HAQMMECEYQYKFJN8MP16QC41"
+ },
  "packages": [
  {
  "path": "postgresql",
@@ -8,7 +11,7 @@
  "queries": "postgresql/query-building.sql",
  "engine": "postgresql",
  "database": {
- "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/jets"
+ "managed": true
  },
  "rules": [
  "sqlc/db-prepare"

examples/ondeck/sqlc.json

@@ -1,5 +1,8 @@
 {
  "version": "1",
+ "cloud": {
+ "project": "01HAQMMECEYQYKFJN8MP16QC41"
+ },
  "packages": [
  {
  "path": "postgresql",
@@ -8,7 +11,7 @@
  "queries": "postgresql/query",
  "engine": "postgresql",
  "database": {
- "uri": "postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/ondeck"
+ "managed": true
  },
  "rules": [
  "sqlc/db-prepare"

internal/cmd/cmd.go

@@ -37,6 +37,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
 rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)")
 rootCmd.PersistentFlags().BoolP("experimental", "x", false, "DEPRECATED: enable experimental features (default: false)")
 rootCmd.PersistentFlags().Bool("no-remote", false, "disable remote execution (default: false)")
+rootCmd.PersistentFlags().Bool("remote", false, "enable remote execution (default: false)")
 rootCmd.PersistentFlags().Bool("no-database", false, "disable database connections (default: false)")
 
 rootCmd.AddCommand(checkCmd)
@@ -136,17 +137,20 @@ var initCmd = &cobra.Command{
 type Env struct {
 DryRun bool
 Debug opts.Debug
+Remote bool
 NoRemote bool
 NoDatabase bool
 }
 
 func ParseEnv(c *cobra.Command) Env {
 dr := c.Flag("dry-run")
+r := c.Flag("remote")
 nr := c.Flag("no-remote")
 nodb := c.Flag("no-database")
 return Env{
 DryRun: dr != nil && dr.Changed,
 Debug: opts.DebugFromEnv(),
+Remote: r != nil && nr.Value.String() == "true",
 NoRemote: nr != nil && nr.Value.String() == "true",
 NoDatabase: nodb != nil && nodb.Value.String() == "true",
 }

internal/cmd/generate.go

@@ -145,7 +145,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
 return nil, err
 }
 
-if conf.Cloud.Project != "" && !e.NoRemote {
+if conf.Cloud.Project != "" && e.Remote && !e.NoRemote {
 return remoteGenerate(ctx, configPath, conf, dir, stderr)
 }
 

internal/cmd/vet.go

@@ -26,7 +26,10 @@ import (
 "github.com/sqlc-dev/sqlc/internal/debug"
 "github.com/sqlc-dev/sqlc/internal/opts"
 "github.com/sqlc-dev/sqlc/internal/plugin"
+"github.com/sqlc-dev/sqlc/internal/quickdb"
+pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 "github.com/sqlc-dev/sqlc/internal/shfmt"
+"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 "github.com/sqlc-dev/sqlc/internal/vet"
 )
 
@@ -376,6 +379,64 @@ type checker struct {
 Envmap map[string]string
 Stderr io.Writer
 NoDatabase bool
+Client pb.QuickClient
+}
+
+func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, func() error, error) {
+cleanup := func() error {
+return nil
+}
+
+if s.Database == nil {
+panic("fetch database URI called with nil database")
+}
+if !s.Database.Managed {
+uri, err := c.DSN(s.Database.URI)
+return uri, cleanup, err
+}
+if s.Engine != config.EnginePostgreSQL {
+return "", cleanup, fmt.Errorf("managed: only PostgreSQL currently")
+}
+
+if c.Client == nil {
+// FIXME: Eventual race condition
+client, err := quickdb.NewClientFromConfig(c.Conf.Cloud)
+if err != nil {
+return "", cleanup, fmt.Errorf("managed: client: %w", err)
+}
+c.Client = client
+}
+
+var migrations []string
+files, err := sqlpath.Glob(s.Schema)
+if err != nil {
+return "", cleanup, err
+}
+for _, query := range files {
+contents, err := os.ReadFile(query)
+if err != nil {
+return "", cleanup, fmt.Errorf("read file: %w", err)
+}
+migrations = append(migrations, string(contents))
+}
+
+resp, err := c.Client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{
+Engine: "postgresql",
+Region: quickdb.GetClosestRegion(),
+Migrations: migrations,
+})
+if err != nil {
+return "", cleanup, fmt.Errorf("managed: create database: %w", err)
+}
+
+cleanup = func() error {
+_, err := c.Client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{
+DatabaseId: resp.DatabaseId,
+})
+return err
+}
+
+return resp.Uri, cleanup, nil
 }
 
 func (c *checker) DSN(dsn string) (string, error) {
@@ -422,10 +483,16 @@ func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
 if c.NoDatabase {
 return fmt.Errorf("database: connections disabled via command line flag")
 }
-dburl, err := c.DSN(s.Database.URI)
+dburl, cleanup, err := c.fetchDatabaseUri(ctx, s)
 if err != nil {
 return err
 }
+defer func() {
+if err := cleanup(); err != nil {
+fmt.Fprintf(c.Stderr, "error cleaning up: %s\n", err)
+}
+}()
+
 switch s.Engine {
 case config.EnginePostgreSQL:
 conn, err := pgx.Connect(ctx, dburl)