From 6d82c4e2ab937b85c84acf56e58c67caea78cb68 Mon Sep 17 00:00:00 2001 From: Robert Williams <1266467+bobdoah@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:42:18 +0100 Subject: [PATCH] feat(wasm): add support for oci urls --- docs/reference/config.md | 16 +- go.mod | 3 + go.sum | 6 + internal/ext/wasm/oci.go | 122 +++++++++++++++ internal/ext/wasm/oci_test.go | 281 ++++++++++++++++++++++++++++++++++ internal/ext/wasm/wasm.go | 8 + 6 files changed, 428 insertions(+), 8 deletions(-) create mode 100644 internal/ext/wasm/oci.go create mode 100644 internal/ext/wasm/oci_test.go diff --git a/docs/reference/config.md b/docs/reference/config.md index 15fdb7cfd4..bccd759aa6 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -14,7 +14,7 @@ sql: queries: "postgresql/query.sql" engine: "postgresql" gen: - go: + go: package: "authors" out: "postgresql" database: @@ -122,7 +122,7 @@ The `analyzer` mapping supports the following keys: - `database`: - If false, do not use the configured database for query analysis. Defaults to `true`. - + ### gen The `gen` mapping supports the following keys: @@ -254,21 +254,21 @@ Each mapping in the `plugins` collection has the following keys: - The format expected. Supports `json` and `protobuf` formats. Defaults to `protobuf`. - `wasm`: A mapping with a two keys `url` and `sha256` - `url`: - - The URL to fetch the WASM file. Supports the `https://` or `file://` schemes. + - The URL to fetch the WASM file. Supports the `https://`, `file://`, or `oci://` schemes. OCI artifacts may be referenced by tag or digest and must use the [CNCF Wasm OCI artifact layout](https://tag-runtime.cncf.io/wgs/wasm/deliverables/wasm-oci-artifact/): exactly one `application/wasm` layer and a config with media type `application/vnd.wasm.config.v0+json`, `architecture: "wasm"`, `os: "wasip1"`, and `layerDigests` matching the layer digest. WASI Preview 2 components are not supported. Registry authentication uses the Docker config, including configured credential helpers. - `sha256` - The SHA256 checksum for the downloaded file. - + ```yaml version: "2" plugins: - name: "py" - wasm: + wasm: url: "https://github.com/sqlc-dev/sqlc-gen-python/releases/download/v0.16.0-alpha/sqlc-gen-python.wasm" sha256: "428476c7408fd4c032da4ec74e8a7344f4fa75e0f98a5a3302f238283b9b95f2" - name: "js" env: - PATH - process: + process: cmd: "sqlc-gen-json" ``` @@ -285,7 +285,7 @@ Each mapping in the `rules` collection has the following keys: See the [vet](../howto/vet.md) documentation for a list of built-in rules and help writing custom rules. - + ```yaml version: "2" sql: @@ -319,7 +319,7 @@ rules: rule: | query.cmd == "exec" ``` - + ### Global overrides Sometimes, the same configuration must be done across various specifications of diff --git a/go.mod b/go.mod index 63324a116a..efcf49c3bf 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/jinzhu/inflection v1.0.0 github.com/ncruces/go-sqlite3 v0.35.6 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.1 github.com/riza-io/grpc-go v0.2.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 @@ -31,6 +33,7 @@ require ( google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 + oras.land/oras-go/v2 v2.6.2 ) require ( diff --git a/go.sum b/go.sum index 014545f708..bd49eb8cae 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,10 @@ github.com/ncruces/go-sqlite3-wasm/v6 v6.3.35304 h1:dBSZlcEFdtBMvNRg34y50mConBPO github.com/ncruces/go-sqlite3-wasm/v6 v6.3.35304/go.mod h1:YvoJzbJpX6phd3BGdtiXu2NuD5RX6G8zsUdzt47GgOY= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/riza-io/grpc-go v0.2.0 h1:2HxQKFVE7VuYstcJ8zqpN84VnAoJ4dCL6YFhJewNcHQ= @@ -114,3 +118,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= diff --git a/internal/ext/wasm/oci.go b/internal/ext/wasm/oci.go new file mode 100644 index 0000000000..2142838ae2 --- /dev/null +++ b/internal/ext/wasm/oci.go @@ -0,0 +1,122 @@ +package wasm + +import ( + "context" + "encoding/json" + "fmt" + "runtime" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/credentials" + "oras.land/oras-go/v2/registry/remote/retry" + + "github.com/sqlc-dev/sqlc/internal/info" +) + +const ( + wasmLayerMediaType = "application/wasm" + wasmConfigMediaType = "application/vnd.wasm.config.v0+json" +) + +type wasmConfig struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + LayerDigests []string `json:"layerDigests"` +} + +func fetchOCI(ctx context.Context, uri string, plainHTTP bool) ([]byte, error) { + reference := strings.TrimPrefix(uri, "oci://") + repo, err := remote.NewRepository(reference) + if err != nil { + return nil, fmt.Errorf("parse OCI reference %q: %w", uri, err) + } + repo.PlainHTTP = plainHTTP + + store, err := credentials.NewStoreFromDocker(credentials.StoreOptions{}) + if err != nil { + return nil, fmt.Errorf("load Docker credentials: %w", err) + } + client := &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: credentials.Credential(store), + } + client.SetUserAgent(fmt.Sprintf("sqlc/%s Go/%s (%s %s)", info.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)) + repo.Client = client + + desc, manifestReader, err := repo.FetchReference(ctx, repo.Reference.Reference) + if err != nil { + return nil, fmt.Errorf("fetch OCI manifest %q: %w", uri, err) + } + defer manifestReader.Close() + if desc.MediaType != ocispec.MediaTypeImageManifest { + return nil, fmt.Errorf("OCI reference %q resolved to unsupported media type %q", uri, desc.MediaType) + } + manifestData, err := content.ReadAll(manifestReader, desc) + if err != nil { + return nil, fmt.Errorf("read OCI manifest %q: %w", uri, err) + } + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestData, &manifest); err != nil { + return nil, fmt.Errorf("decode OCI manifest %q: %w", uri, err) + } + if manifest.SchemaVersion != 2 { + return nil, fmt.Errorf("OCI artifact %q: unsupported schema version %d", uri, manifest.SchemaVersion) + } + layer, err := wasmLayer(manifest.Layers) + if err != nil { + return nil, fmt.Errorf("OCI artifact %q: %w", uri, err) + } + if manifest.Config.MediaType != wasmConfigMediaType { + return nil, fmt.Errorf("OCI artifact %q: unsupported config media type %q", uri, manifest.Config.MediaType) + } + configData, err := content.FetchAll(ctx, repo, manifest.Config) + if err != nil { + return nil, fmt.Errorf("fetch OCI WASM config %q: %w", uri, err) + } + if err := validateWASMConfig(configData, layer); err != nil { + return nil, fmt.Errorf("OCI artifact %q: %w", uri, err) + } + + body, err := repo.Fetch(ctx, layer) + if err != nil { + return nil, fmt.Errorf("fetch OCI WASM layer %q: %w", uri, err) + } + defer body.Close() + wmod, err := content.ReadAll(body, layer) + if err != nil { + return nil, fmt.Errorf("read OCI WASM layer %q: %w", uri, err) + } + return wmod, nil +} + +func wasmLayer(layers []ocispec.Descriptor) (ocispec.Descriptor, error) { + if len(layers) != 1 { + return ocispec.Descriptor{}, fmt.Errorf("manifest must contain exactly one WASM layer") + } + if layers[0].MediaType != wasmLayerMediaType { + return ocispec.Descriptor{}, fmt.Errorf("unsupported WASM layer media type %q", layers[0].MediaType) + } + return layers[0], nil +} + +func validateWASMConfig(data []byte, layer ocispec.Descriptor) error { + var config wasmConfig + if err := json.Unmarshal(data, &config); err != nil { + return fmt.Errorf("decode WASM config: %w", err) + } + if config.Architecture != "wasm" { + return fmt.Errorf("unsupported WASM architecture %q", config.Architecture) + } + if config.OS != "wasip1" { + return fmt.Errorf("unsupported WASM OS %q: only wasip1 plugins are supported", config.OS) + } + if len(config.LayerDigests) != 1 || config.LayerDigests[0] != layer.Digest.String() { + return fmt.Errorf("WASM config layerDigests must match the manifest layers") + } + return nil +} diff --git a/internal/ext/wasm/oci_test.go b/internal/ext/wasm/oci_test.go new file mode 100644 index 0000000000..86dc91e5e7 --- /dev/null +++ b/internal/ext/wasm/oci_test.go @@ -0,0 +1,281 @@ +package wasm + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestFetchOCIWithDockerCredentials(t *testing.T) { + wasm := []byte("\x00asm\x01\x00\x00\x00") + layer := ocispec.Descriptor{ + MediaType: wasmLayerMediaType, + Digest: digest.FromBytes(wasm), + Size: int64(len(wasm)), + } + configData := []byte(fmt.Sprintf(`{"architecture":"wasm","os":"wasip1","layerDigests":[%q]}`, layer.Digest.String())) + configDigest := digest.FromBytes(configData) + manifest, err := json.Marshal(ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispec.MediaTypeImageManifest, + Config: ocispec.Descriptor{ + MediaType: wasmConfigMediaType, + Digest: configDigest, + Size: int64(len(configData)), + }, + Layers: []ocispec.Descriptor{layer}, + }) + if err != nil { + t.Fatal(err) + } + manifestDigest := digest.FromBytes(manifest) + servedManifest := manifest + servedWASM := wasm + servedConfig := configData + + const username = "sqlc" + const password = "secret" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUser, gotPassword, ok := r.BasicAuth() + if !ok || gotUser != username || gotPassword != password { + w.Header().Set("WWW-Authenticate", `Basic realm="registry"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + switch r.URL.Path { + case "/v2/plugins/sqlc/manifests/v1", "/v2/plugins/sqlc/manifests/" + manifestDigest.String(): + w.Header().Set("Content-Type", ocispec.MediaTypeImageManifest) + w.Header().Set("Docker-Content-Digest", manifestDigest.String()) + _, _ = w.Write(servedManifest) + case "/v2/plugins/sqlc/blobs/" + layer.Digest.String(): + w.Header().Set("Content-Type", wasmLayerMediaType) + w.Header().Set("Docker-Content-Digest", layer.Digest.String()) + _, _ = w.Write(servedWASM) + case "/v2/plugins/sqlc/blobs/" + configDigest.String(): + w.Header().Set("Content-Type", wasmConfigMediaType) + w.Header().Set("Docker-Content-Digest", configDigest.String()) + _, _ = w.Write(servedConfig) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + dockerConfig := t.TempDir() + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + config := fmt.Sprintf(`{"auths":{%q:{"auth":%q}}}`, serverURL.Host, auth) + if err := os.WriteFile(filepath.Join(dockerConfig, "config.json"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("DOCKER_CONFIG", dockerConfig) + + // Keep the advertised digests and lengths unchanged so only verification + // of the response bytes can detect the corruption. + corruptWASM := bytes.Clone(wasm) + corruptWASM[len(corruptWASM)-1] ^= 1 + tests := []struct { + name string + reference string + manifest []byte + wasm []byte + config []byte + wantErr string + }{ + { + name: "valid artifact", + manifest: manifest, + wasm: wasm, + }, + { + name: "digest reference", + reference: "@" + manifestDigest.String(), + manifest: manifest, + wasm: wasm, + }, + { + name: "corrupt manifest", + manifest: bytes.Replace(manifest, []byte("wasm"), []byte("xasm"), 1), + wasm: wasm, + wantErr: "read OCI manifest", + }, + { + name: "corrupt config", + manifest: manifest, + wasm: wasm, + config: bytes.Replace(configData, []byte("wasip1"), []byte("wasip2"), 1), + wantErr: "fetch OCI WASM config", + }, + { + name: "corrupt WASM layer", + manifest: manifest, + wasm: corruptWASM, + wantErr: "read OCI WASM layer", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + servedManifest = tt.manifest + servedWASM = tt.wasm + servedConfig = configData + if tt.config != nil { + servedConfig = tt.config + } + reference := tt.reference + if reference == "" { + reference = ":v1" + } + got, err := fetchOCI(context.Background(), "oci://"+serverURL.Host+"/plugins/sqlc"+reference, true) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if gotSum, wantSum := sha256.Sum256(got), sha256.Sum256(wasm); gotSum != wantSum { + t.Fatalf("WASM checksum = %x, want %x", gotSum, wantSum) + } + }) + } +} + +func TestWASMLayer(t *testing.T) { + wasm := ocispec.Descriptor{MediaType: wasmLayerMediaType, Digest: digest.FromString("wasm")} + other := ocispec.Descriptor{MediaType: "application/octet-stream", Digest: digest.FromString("other")} + tests := []struct { + name string + layers []ocispec.Descriptor + want ocispec.Descriptor + wantErr bool + }{ + { + name: "single WASM layer", + layers: []ocispec.Descriptor{wasm}, + want: wasm, + }, + { + name: "unsupported layer media type", + layers: []ocispec.Descriptor{other}, + wantErr: true, + }, + { + name: "multiple WASM layers", + layers: []ocispec.Descriptor{wasm, wasm}, + wantErr: true, + }, + { + name: "WASM layer in multi-layer artifact", + layers: []ocispec.Descriptor{other, wasm}, + wantErr: true, + }, + { + name: "no layers", + wantErr: true, + }, + { + name: "multi-layer artifact without WASM", + layers: []ocispec.Descriptor{other, other}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := wasmLayer(tt.layers) + if (err != nil) != tt.wantErr { + t.Fatalf("wasmLayer() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if got.MediaType != tt.want.MediaType || got.Digest != tt.want.Digest { + t.Fatalf("wasmLayer() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidateWASMConfig(t *testing.T) { + layer := ocispec.Descriptor{MediaType: wasmLayerMediaType, Digest: digest.FromString("wasm")} + tests := []struct { + name string + data string + wantErr string + }{ + { + name: "WASI Preview 1", + data: fmt.Sprintf(`{"architecture":"wasm","os":"wasip1","layerDigests":[%q]}`, layer.Digest.String()), + }, + { + name: "invalid JSON", + data: `{`, + wantErr: "decode WASM config", + }, + { + name: "unsupported architecture", + data: `{"architecture":"amd64","os":"wasip1"}`, + wantErr: "unsupported WASM architecture", + }, + { + name: "missing architecture", + data: `{"os":"wasip1"}`, + wantErr: "unsupported WASM architecture", + }, + { + name: "WASI Preview 2 component", + data: `{"architecture":"wasm","os":"wasip2","component":{"imports":[],"exports":[]}}`, + wantErr: "only wasip1 plugins are supported", + }, + { + name: "missing OS", + data: `{"architecture":"wasm"}`, + wantErr: "unsupported WASM OS", + }, + { + name: "missing layer digests", + data: `{"architecture":"wasm","os":"wasip1"}`, + wantErr: "layerDigests must match", + }, + { + name: "mismatched layer digest", + data: fmt.Sprintf(`{"architecture":"wasm","os":"wasip1","layerDigests":[%q]}`, digest.FromString("other")), + wantErr: "layerDigests must match", + }, + { + name: "extra layer digest", + data: fmt.Sprintf(`{"architecture":"wasm","os":"wasip1","layerDigests":[%q,%q]}`, layer.Digest, layer.Digest), + wantErr: "layerDigests must match", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateWASMConfig([]byte(tt.data), layer) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + } else if err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go index e313ac29ff..3e43f34e16 100644 --- a/internal/ext/wasm/wasm.go +++ b/internal/ext/wasm/wasm.go @@ -96,6 +96,14 @@ func (r *Runner) fetch(ctx context.Context, uri string) ([]byte, string, error) } body = resp.Body + case strings.HasPrefix(uri, "oci://"): + wmod, err := fetchOCI(ctx, uri, false) + if err != nil { + return nil, "", err + } + sum := sha256.Sum256(wmod) + return wmod, fmt.Sprintf("%x", sum), nil + default: return nil, "", fmt.Errorf("unknown scheme: %s", r.URL) }