Skip to content
38 changes: 15 additions & 23 deletions docs/howto/upload.md
Original file line number Diff line number Diff line change
@@ -1,35 +1,26 @@
# Uploading projects

*This feature requires signing up for [sqlc Cloud](https://app.sqlc.dev), which is currently in beta.*
*Added in v1.22.0*

Uploading your project ensures that future releases of sqlc do not break your
existing code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
project, uploaded projects are tested against development releases of sqlc to
Uploading an archive of your project ensures that future releases of sqlc do not
break your code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
project, uploaded archives are tested against development releases of sqlc to
verify correctness.

Interested in uploading projects? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email
at [hello@sqlc.dev](mailto:hello@sqlc.dev).

## Add configuration

After creating a project, add the project ID to your sqlc configuration file.

```yaml
version: "1"
project:
id: "<PROJECT-ID>"
packages: []
```

```json
{
"version": "1",
"project": {
"id": "<PROJECT-ID>"
},
"packages": [
]
}
version: "2"
cloud:
project: "<PROJECT-ID>"
```

You'll also need to create an API token and make it available via the
You'll also need to create an auth token and make it available via the
`SQLC_AUTH_TOKEN` environment variable.

```shell
Expand All @@ -38,13 +29,14 @@ export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx

## Dry run

You can see what's included when uploading your project by using using the `--dry-run` flag:
You can see what's included when uploading your project by using using the
`--dry-run` flag:

```shell
sqlc upload --dry-run
```

The output will be the exact HTTP request sent by `sqlc`.
The output is the request `sqlc` would have sent without the `--dry-run` flag.

## Upload

Expand All @@ -54,4 +46,4 @@ Once you're ready to upload, remove the `--dry-run` flag.
sqlc upload
```

By uploading your project, you're making sqlc more stable and reliable. Thanks!
By uploading your project, you're making sqlc more stable and reliable. Thanks!
16 changes: 0 additions & 16 deletions internal/bundler/metadata.go

This file was deleted.

65 changes: 20 additions & 45 deletions internal/bundler/multipart.go
Original file line number Diff line number Diff line change
@@ -1,80 +1,55 @@
package bundler

import (
"io"
"mime/multipart"
"os"
"path/filepath"

"github.com/sqlc-dev/sqlc/internal/config"
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
)

func writeInputs(w *multipart.Writer, file string, conf *config.Config) error {
func readInputs(file string, conf *config.Config) ([]*pb.File, error) {
refs := map[string]struct{}{}
refs[filepath.Base(file)] = struct{}{}

for _, pkg := range conf.SQL {
for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
files, err := sqlpath.Glob(paths)
if err != nil {
return err
return nil, err
}
for _, file := range files {
refs[file] = struct{}{}
}
}
}

var files []*pb.File
for file, _ := range refs {
if err := addPart(w, file); err != nil {
return err
}
}

params, err := projectMetadata()
if err != nil {
return err
}
params = append(params, [2]string{"project_id", conf.Project.ID})
for _, val := range params {
if err = w.WriteField(val[0], val[1]); err != nil {
return err
contents, err := os.ReadFile(file)
if err != nil {
return nil, err
}
files = append(files, &pb.File{
Name: file,
Contents: contents,
})
}
return nil
return files, nil
}

func addPart(w *multipart.Writer, file string) error {
h, err := os.Open(file)
if err != nil {
return err
}
defer h.Close()
part, err := w.CreateFormFile("inputs", file)
if err != nil {
return err
}
_, err = io.Copy(part, h)
if err != nil {
return err
}
return nil
}

func writeOutputs(w *multipart.Writer, dir string, output map[string]string) error {
func readOutputs(dir string, output map[string]string) ([]*pb.File, error) {
var files []*pb.File
for filename, contents := range output {
rel, err := filepath.Rel(dir, filename)
if err != nil {
return err
}
part, err := w.CreateFormFile("outputs", rel)
if err != nil {
return err
}
if _, err := io.WriteString(part, contents); err != nil {
return err
return nil, err
}
files = append(files, &pb.File{
Name: rel,
Contents: contents,
})
}
return nil
return files, nil
}
66 changes: 27 additions & 39 deletions internal/bundler/upload.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,24 @@
package bundler

import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httputil"
"os"

"google.golang.org/protobuf/encoding/protojson"

"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/info"
"github.com/sqlc-dev/sqlc/internal/quickdb"
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
)

type Uploader struct {
token string
configPath string
config *config.Config
dir string
client pb.QuickClient
}

func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
Expand All @@ -30,47 +31,44 @@ func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
}

func (up *Uploader) Validate() error {
if up.config.Project.ID == "" {
return fmt.Errorf("project.id is not set")
if up.config.Cloud.Project == "" {
return fmt.Errorf("cloud.project is not set")
}
if up.token == "" {
return fmt.Errorf("SQLC_AUTH_TOKEN environment variable is not set")
}
if up.client == nil {
client, err := quickdb.NewClientFromConfig(up.config.Cloud)
if err != nil {
return fmt.Errorf("client init failed: %w", err)
}
up.client = client
}
return nil
}

func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*http.Request, error) {
body := bytes.NewBuffer([]byte{})
w := multipart.NewWriter(body)
if err := writeInputs(w, up.configPath, up.config); err != nil {
return nil, err
}
if err := writeOutputs(w, up.dir, result); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*pb.UploadArchiveRequest, error) {
ins, err := readInputs(up.configPath, up.config)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", "https://api.sqlc.dev/upload", body)
outs, err := readOutputs(up.dir, result)
if err != nil {
return nil, err
}
// Set sqlc-version header
req.Header.Set("Content-Type", w.FormDataContentType())
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", up.token))
return req.WithContext(ctx), nil
return &pb.UploadArchiveRequest{
SqlcVersion: info.Version,
Inputs: ins,
Outputs: outs,
}, nil
}

func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error {
req, err := up.buildRequest(ctx, result)
if err != nil {
return err
}
dump, err := httputil.DumpRequest(req, true)
if err != nil {
return err
}
os.Stdout.Write(dump)
fmt.Println(protojson.Format(req))
return nil
}

Expand All @@ -82,18 +80,8 @@ func (up *Uploader) Upload(ctx context.Context, result map[string]string) error
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
body, err := io.ReadAll(resp.Body)
defer resp.Body.Close()
if err != nil {
return fmt.Errorf("upload error: endpoint returned non-200 status code: %d", resp.StatusCode)
}
return fmt.Errorf("upload error: %d: %s", resp.StatusCode, string(body))
if _, err := up.client.UploadArchive(ctx, req); err != nil {
return fmt.Errorf("upload error: %w", err)
}
return nil
}
5 changes: 0 additions & 5 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,13 @@ const (

type Config struct {
Version string `json:"version" yaml:"version"`
Project Project `json:"project" yaml:"project"`
Cloud Cloud `json:"cloud" yaml:"cloud"`
SQL []SQL `json:"sql" yaml:"sql"`
Gen Gen `json:"overrides,omitempty" yaml:"overrides"`
Plugins []Plugin `json:"plugins" yaml:"plugins"`
Rules []Rule `json:"rules" yaml:"rules"`
}

type Project struct {
ID string `json:"id" yaml:"id"`
}

type Database struct {
URI string `json:"uri" yaml:"uri"`
Managed bool `json:"managed" yaml:"managed"`
Expand Down
2 changes: 0 additions & 2 deletions internal/config/v_one.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
type V1GenerateSettings struct {
Version string `json:"version" yaml:"version"`
Cloud Cloud `json:"cloud" yaml:"cloud"`
Project Project `json:"project" yaml:"project"`
Packages []v1PackageSettings `json:"packages" yaml:"packages"`
Overrides []Override `json:"overrides,omitempty" yaml:"overrides,omitempty"`
Rename map[string]string `json:"rename,omitempty" yaml:"rename,omitempty"`
Expand Down Expand Up @@ -132,7 +131,6 @@ func (c *V1GenerateSettings) ValidateGlobalOverrides() error {
func (c *V1GenerateSettings) Translate() Config {
conf := Config{
Version: c.Version,
Project: c.Project,
Cloud: c.Cloud,
Rules: c.Rules,
}
Expand Down
8 changes: 0 additions & 8 deletions internal/config/v_one.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,6 @@
"version": {
"const": "1"
},
"project": {
"type": "object",
"properties": {
"id": {
"type": "string"
}
}
},
"cloud": {
"type": "object",
"properties": {
Expand Down
8 changes: 0 additions & 8 deletions internal/config/v_two.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,6 @@
"version": {
"const": "2"
},
"project": {
"type": "object",
"properties": {
"id": {
"type": "string"
}
}
},
"cloud": {
"type": "object",
"properties": {
Expand Down
Loading