diff --git a/cmd/crossplane/composition/generate.go b/cmd/crossplane/composition/generate.go index dbf7c797..d1874890 100644 --- a/cmd/crossplane/composition/generate.go +++ b/cmd/crossplane/composition/generate.go @@ -19,6 +19,7 @@ package composition import ( "context" "fmt" + "os" "path/filepath" "strings" @@ -121,9 +122,15 @@ func (c *generateCmd) AfterApply(cfg *config.Config) error { func (c *generateCmd) Run(sp terminal.SpinnerPrinter) error { ctx := context.Background() - if err := sp.WrapWithSuccessSpinner("Ensuring function-auto-ready dependency", func() error { + err := sp.WrapWithSuccessSpinner("Ensuring function-auto-ready dependency", func() error { return c.ensureFunctionAutoReady(ctx) - }); err != nil { + }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + if c.depManager != nil { + c.depManager.PrintWarnings(os.Stderr) + } + if err != nil { return errors.Wrap(err, "failed to ensure function-auto-ready dependency") } diff --git a/cmd/crossplane/dependency/add.go b/cmd/crossplane/dependency/add.go index 6424f473..74733a4f 100644 --- a/cmd/crossplane/dependency/add.go +++ b/cmd/crossplane/dependency/add.go @@ -18,6 +18,7 @@ package dependency import ( "context" + "os" "path/filepath" "strings" @@ -108,9 +109,13 @@ func (c *addCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con desc := dependency.GetSourceDescription(dep) logger.Debug("Adding dependency", "dependency", desc) - return sp.WrapWithSuccessSpinner("Adding "+desc, func() error { + err = sp.WrapWithSuccessSpinner("Adding "+desc, func() error { return m.AddDependency(ctx, &dep) }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + m.PrintWarnings(os.Stderr) + return err } func (c *addCmd) buildDependency() (v1alpha1.Dependency, error) { diff --git a/cmd/crossplane/dependency/cache.go b/cmd/crossplane/dependency/cache.go index 3cecc882..c5f53ea3 100644 --- a/cmd/crossplane/dependency/cache.go +++ b/cmd/crossplane/dependency/cache.go @@ -19,6 +19,7 @@ package dependency import ( "context" "fmt" + "os" "path/filepath" "github.com/alecthomas/kong" @@ -106,9 +107,13 @@ func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, m := dependency.NewManager(proj, projFS, opts...) logger.Debug("Updating all dependencies") - return sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { + err = sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { return m.RefreshAll(ctx, ch) }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + m.PrintWarnings(os.Stderr) + return err } //go:embed help/clean-cache.md diff --git a/cmd/crossplane/function/generate.go b/cmd/crossplane/function/generate.go index cb7585a4..8b3ae64d 100644 --- a/cmd/crossplane/function/generate.go +++ b/cmd/crossplane/function/generate.go @@ -23,6 +23,7 @@ import ( "embed" "fmt" "io/fs" + "os" "path" "path/filepath" "slices" @@ -40,6 +41,7 @@ import ( v1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/kcl" "github.com/crossplane/cli/v2/internal/project/projectfile" @@ -47,6 +49,7 @@ import ( "github.com/crossplane/cli/v2/internal/schemas/manager" "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" + clixpkg "github.com/crossplane/cli/v2/internal/xpkg" ) //go:embed help/generate.md @@ -117,9 +120,39 @@ func (c *generateCmd) AfterApply() error { c.fsPath = path.Join(proj.Spec.Paths.Functions, c.Name) c.projectRepository = proj.Spec.Repository c.projectSource = proj.Spec.Source + return nil } +// newDependencyManager builds the dependency manager used to generate +// schemas in Run, mirroring the project build command's wiring. +func (c *generateCmd) newDependencyManager(cfg *config.Config) (*dependency.Manager, error) { + cacheDir := dependency.DefaultCacheDir() + schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) + generators := generator.Filter( + generator.AllLanguages( + generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), + generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), + ), + c.proj.Spec.Schemas.GetLanguages(), + ) + client, err := clixpkg.NewClient( + clixpkg.NewRemoteFetcher(), + clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), + clixpkg.WithImageConfigs(c.proj.Spec.ImageConfigs), + ) + if err != nil { + return nil, errors.Wrap(err, "cannot set up the package client that fetches the project's dependencies") + } + return dependency.NewManager(c.proj, c.projFS, + dependency.WithProjectFile(filepath.Base(c.ProjectFile)), + dependency.WithSchemaFS(schemasFS), + dependency.WithSchemaGenerators(generators), + dependency.WithXpkgClient(client), + dependency.WithResolver(clixpkg.NewResolver(client)), + ), nil +} + // validateLanguageAgainstSchemas refuses to generate a function in a language // whose schemas the project doesn't generate. Such a function would have no // models to import, which is surprising, so we fail up front rather than @@ -154,10 +187,6 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error } ctx := context.Background() - apisFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.APIs) - if c.proj.Spec.Paths.APIs == "/" { - apisFS = c.projFS - } schemaMgr := manager.New( c.schemasFS, generator.Filter( @@ -170,10 +199,31 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)), ) - if err := sp.WrapWithSuccessSpinner("Generating schemas", func() error { - _, err := schemaMgr.Generate(ctx, manager.NewFSSource(c.proj.Spec.Paths.APIs, apisFS)) - return err - }); err != nil { + var depMgr *dependency.Manager + err := sp.WrapWithSuccessSpinner("Generating schemas", func() error { + var err error + depMgr, err = c.newDependencyManager(cfg) + if err != nil { + return err + } + + // Unlike before this PR, generating a function now collects every + // dependency declared in the project file, like project build does: + // the generated function's models need the dependencies, not just + // apis/. The first run in a project needs network access (or a warm + // per-user package cache); later runs reuse the project's caches. + depSources, err := depMgr.CollectSources(ctx, nil) + if err != nil { + return errors.Wrap(err, "cannot fetch the project's dependencies, whose models the function needs; check network access and registry credentials") + } + return schemaMgr.GenerateFromMultipleSources(ctx, depSources, manager.NewProjectAPIsSource(c.projFS, c.proj.Spec.Paths)) + }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + if depMgr != nil { + depMgr.PrintWarnings(os.Stderr) + } + if err != nil { return errors.Wrap(err, "failed to generate schemas") } diff --git a/cmd/crossplane/function/help/generate.md b/cmd/crossplane/function/help/generate.md index bf0c666b..2df2a45b 100644 --- a/cmd/crossplane/function/help/generate.md +++ b/cmd/crossplane/function/help/generate.md @@ -3,6 +3,12 @@ language under the project's `functions/` directory. It optionally idempotently adds the new function to the end of a Composition's pipeline when given a Composition path. +Before it writes the function, the command generates schemas for the +project's own APIs and for every dependency in the project file, so the +function can use their models. Dependencies that aren't in the per-user +package cache are fetched, so the first run in a project may need network +access and registry credentials. + ## Supported languages The following are valid arguments to the `--language` / `-l` flag: diff --git a/cmd/crossplane/project/build.go b/cmd/crossplane/project/build.go index 06222ae6..2a04c721 100644 --- a/cmd/crossplane/project/build.go +++ b/cmd/crossplane/project/build.go @@ -158,6 +158,9 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *c ) return buildErr }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + depMgr.PrintWarnings(os.Stderr) if err != nil { return err } diff --git a/cmd/crossplane/project/run.go b/cmd/crossplane/project/run.go index 73ba0ad7..886d5895 100644 --- a/cmd/crossplane/project/run.go +++ b/cmd/crossplane/project/run.go @@ -253,6 +253,9 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con return eg.Wait() }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + depMgr.PrintWarnings(os.Stderr) if err != nil { return err } diff --git a/cmd/crossplane/render/op/cmd.go b/cmd/crossplane/render/op/cmd.go index 065eb6a8..533bf6b3 100644 --- a/cmd/crossplane/render/op/cmd.go +++ b/cmd/crossplane/render/op/cmd.go @@ -383,7 +383,7 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal return nil, errors.Wrap(err, "cannot load project functions") } - if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { + buildErr := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) @@ -417,8 +417,12 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal embeddedFns, err := project.EmbeddedFunctionsToDaemon(ctx, imgMap) fns = append(fns, embeddedFns...) return err - }); err != nil { - return nil, errors.Wrap(err, "cannot build embedded functions") + }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + depMgr.PrintWarnings(os.Stderr) + if buildErr != nil { + return nil, errors.Wrap(buildErr, "cannot build embedded functions") } return fns, nil diff --git a/cmd/crossplane/render/xr/cmd.go b/cmd/crossplane/render/xr/cmd.go index 10cd4332..81de5cce 100644 --- a/cmd/crossplane/render/xr/cmd.go +++ b/cmd/crossplane/render/xr/cmd.go @@ -495,7 +495,7 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger, return nil, errors.Wrap(err, "cannot load project functions") } - if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { + buildErr := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) @@ -526,8 +526,12 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger, embeddedFns, err := project.EmbeddedFunctionsToDaemon(ctx, imgMap) fns = append(fns, embeddedFns...) return err - }); err != nil { - return nil, errors.Wrap(err, "cannot build embedded functions") + }) + // Collection warnings are printed once the spinner has stopped, failed or + // not: a warning can explain the failure. + depMgr.PrintWarnings(os.Stderr) + if buildErr != nil { + return nil, errors.Wrap(buildErr, "cannot build embedded functions") } return fns, nil diff --git a/internal/dependency/manager.go b/internal/dependency/manager.go index 6e34c251..95527404 100644 --- a/internal/dependency/manager.go +++ b/internal/dependency/manager.go @@ -21,6 +21,9 @@ package dependency import ( "context" "fmt" + "io" + "maps" + "slices" "strings" "sync" @@ -62,10 +65,30 @@ type Manager struct { updateMutex sync.Mutex + // fetched caches packages by resolved ref for this Manager, so the + // top-level walks -- GVK extraction, then source collection -- do not + // each download the same package. The xpkg client's own cache is + // per-invocation only when configured, and is keyed for extraction, not + // for the GVK and dependency walk both callers need. + fetched map[string]*runtimexpkg.Package + fetchedMu sync.Mutex + // visited tracks package refs already processed during this invocation, // deduplicating shared transitive dependencies and breaking cycles. visited map[string]struct{} visitedMu sync.Mutex + + // collected records, per package repository, every version source + // collection resolved it to, so CollectSources can warn when the + // project's dependencies pull in one package at two versions. Guarded by + // visitedMu and reset with visited. + collected map[string][]string + + // warnings are the non-fatal problems the last source collection found. + // They are kept rather than printed, because collection runs under a + // spinner that owns the terminal; see PrintWarnings. Guarded by + // visitedMu and reset with visited. + warnings []string } // ManagerOption configures the dependency manager. @@ -167,6 +190,8 @@ func NewManager(proj *v1alpha1.Project, projFS afero.Fs, opts ...ManagerOption) client: options.client, resolver: options.resolver, visited: make(map[string]struct{}), + collected: make(map[string][]string), + fetched: make(map[string]*runtimexpkg.Package), } } @@ -178,30 +203,26 @@ func (m *Manager) ResolveRef(ref string) (name.Reference, error) { return resolved, err } -// AddPackage adds a package to the dependency manager. If refresh is set, the -// package's ref will be re-resolved regardless of whether it is cached. -// Schemas are also generated recursively for the dependencies declared in the -// package's metadata. -func (m *Manager) AddPackage(ctx context.Context, ref string, refresh bool) (*schema.GroupVersionKind, error) { - return m.addPackage(ctx, ref, refresh) -} - +// addPackage fetches a package by ref -- walking its transitive dependencies +// -- and returns its runtime GVK. It writes nothing: callers that add a +// dependency to the project finish with GenerateAllSchemas so the tree always +// describes the whole project. func (m *Manager) addPackage(ctx context.Context, ref string, refresh bool) (*schema.GroupVersionKind, error) { // Record the ref unconditionally: top-level calls must always fully // process the package so callers get its GVK, while transitive walks // consult the visited set before recursing. m.claim(ref) - resolvedRef, version, err := m.resolver.Resolve(ctx, ref) + resolvedRef, _, err := m.resolver.Resolve(ctx, ref) if err != nil { - return nil, errors.Wrapf(err, "cannot resolve %s", ref) + return nil, errors.Wrapf(err, "cannot resolve package %q; check that the package exists and that the version or digest is valid", ref) } pullPolicy := corev1.PullIfNotPresent if refresh { pullPolicy = corev1.PullAlways } - pkg, err := m.client.Get(ctx, resolvedRef.String(), runtimexpkg.WithPullPolicy(pullPolicy)) + pkg, err := m.getPackage(ctx, resolvedRef.String(), pullPolicy) if err != nil { return nil, errors.Wrapf(err, "failed to fetch %s", ref) } @@ -210,22 +231,6 @@ func (m *Manager) addPackage(ctx context.Context, ref string, refresh bool) (*sc if err != nil { return nil, errors.Wrapf(err, "cannot identify package %s", ref) } - crdFS, err := clixpkg.CRDFilesystem(pkg.Package) - if err != nil { - return nil, errors.Wrapf(err, "cannot extract CRDs from %s", ref) - } - - // Use the resolved version so constraint and exact-version inputs - // collapse to one schema-lock entry. Digest-pinned refs (no resolved - // version) intentionally use a digest-form ID and remain separate. - id := pkg.Source + "@" + pkg.Digest - if version != "" { - id = pkg.Source + ":" + version - } - src := smanager.NewXpkgSource(id, pkg.Digest, crdFS) - if err := m.schemas.Add(ctx, src); err != nil { - return nil, err - } if err := m.addTransitiveDeps(ctx, pkg, refresh); err != nil { return nil, err } @@ -252,6 +257,33 @@ func (m *Manager) addTransitiveDeps(ctx context.Context, pkg *runtimexpkg.Packag return nil } +// getPackage fetches a package by resolved ref, memoizing the result so the +// GVK-extraction walk and the source-collection walk of the same top-level +// call do not each download it. A refresh fetch bypasses and replaces the +// memoized package; the current contents are what the ref should resolve to +// now. +func (m *Manager) getPackage(ctx context.Context, resolvedRef string, pullPolicy corev1.PullPolicy) (*runtimexpkg.Package, error) { + if pullPolicy != corev1.PullAlways { + m.fetchedMu.Lock() + pkg, ok := m.fetched[resolvedRef] + m.fetchedMu.Unlock() + if ok { + return pkg, nil + } + } + + pkg, err := m.client.Get(ctx, resolvedRef, runtimexpkg.WithPullPolicy(pullPolicy)) + if err != nil { + return nil, err + } + + m.fetchedMu.Lock() + m.fetched[resolvedRef] = pkg + m.fetchedMu.Unlock() + + return pkg, nil +} + // claim records ref as processed and reports whether it was newly claimed. func (m *Manager) claim(ref string) bool { m.visitedMu.Lock() @@ -263,6 +295,38 @@ func (m *Manager) claim(ref string) bool { return true } +// resetVisited clears the set claim consults, so each top-level traversal +// (RefreshAll, CollectSources) starts with none of its dependencies +// already claimed. Without this, a Manager reused for a second traversal -- +// a retry, or a caller that calls CollectSources more than once -- would see +// every ref the first traversal claimed as already visited and silently skip +// it, rather than traverse it again. +func (m *Manager) resetVisited() { + m.visitedMu.Lock() + defer m.visitedMu.Unlock() + m.visited = make(map[string]struct{}) + m.collected = make(map[string][]string) + m.warnings = nil +} + +// Warnings returns the non-fatal problems the last source collection found, +// such as a package resolved to more than one version. +func (m *Manager) Warnings() []string { + m.visitedMu.Lock() + defer m.visitedMu.Unlock() + return slices.Clone(m.warnings) +} + +// PrintWarnings writes the last source collection's warnings to w, one per +// line. Commands call it once their spinners have stopped: collection runs +// under a spinner, and a warning written to the same terminal mid-spin would +// be garbled by it. +func (m *Manager) PrintWarnings(w io.Writer) { + for _, warning := range m.Warnings() { + _, _ = fmt.Fprintln(w, "warning: "+warning) + } +} + // dependencyRepo returns the OCI repository of a package metadata dependency, // or "" if none is set. func dependencyRepo(dep pkgmetav1.Dependency) string { @@ -310,9 +374,14 @@ func runtimeGVKForPackage(pkg *runtimexpkg.Package) (*schema.GroupVersionKind, e }, nil } -// AddDependency adds a dependency, generates schemas for it, and persists the -// dependency to the project file. +// AddDependency adds a dependency to the project file and regenerates the +// schemas for the whole project from every source together. Schemas are +// generated in one merged pass -- not just for the new dependency -- because +// a per-source write would leave the tree describing a different set of +// sources than the project has. func (m *Manager) AddDependency(ctx context.Context, dep *v1alpha1.Dependency) error { + m.resetVisited() + gvk, err := m.addDependencyNoWrite(ctx, dep, false) if err != nil { return err @@ -329,25 +398,30 @@ func (m *Manager) AddDependency(ctx context.Context, dep *v1alpha1.Dependency) e m.updateMutex.Lock() defer m.updateMutex.Unlock() + // Add the dependency in memory and generate before touching the project + // file. Generation is what fetches git, HTTP and k8s sources, so a + // dependency that cannot be fetched or generated is never persisted. + prev := slices.Clone(m.proj.Spec.Dependencies) upsertDependency(m.proj, *dep) + if err := m.GenerateAllSchemas(ctx, nil); err != nil { + m.proj.Spec.Dependencies = prev + return err + } + + // If this write fails, the schemas describe a dependency the project file + // lacks; the lock then no longer matches the project, so the next pass + // regenerates. return projectfile.Update(m.projFS, m.projFile, func(p *v1alpha1.Project) { upsertDependency(p, *dep) }) } -// AddAll adds all dependencies configured in the project. If ch is non-nil, -// events will be sent for each dependency as it is processed. -func (m *Manager) AddAll(ctx context.Context, ch async.EventChannel) error { - return m.addAll(ctx, ch, false) -} - // RefreshAll re-resolves every dependency's version constraint against the -// registry. Used by `dependency update-cache`. +// registry, re-pulls each package, and regenerates the schemas. Used by +// `dependency update-cache`. func (m *Manager) RefreshAll(ctx context.Context, ch async.EventChannel) error { - return m.addAll(ctx, ch, true) -} + m.resetVisited() -func (m *Manager) addAll(ctx context.Context, ch async.EventChannel, refresh bool) error { eg, egCtx := errgroup.WithContext(ctx) for i := range m.proj.Spec.Dependencies { @@ -355,7 +429,7 @@ func (m *Manager) addAll(ctx context.Context, ch async.EventChannel, refresh boo desc := "Updating dependency " + GetSourceDescription(*dep) eg.Go(func() error { ch.SendEvent(desc, async.EventStatusStarted) - if _, err := m.addDependencyNoWrite(egCtx, dep, refresh); err != nil { + if _, err := m.addDependencyNoWrite(egCtx, dep, true); err != nil { ch.SendEvent(desc, async.EventStatusFailure) return err } @@ -364,9 +438,43 @@ func (m *Manager) addAll(ctx context.Context, ch async.EventChannel, refresh boo }) } - return eg.Wait() + if err := eg.Wait(); err != nil { + return err + } + + // Fetching and re-resolving never writes schemas; finish with one merged + // generation pass over every source so the tree describes the whole + // project, not whichever dependency happened to be processed last. + return m.GenerateAllSchemas(ctx, ch) } +// GenerateAllSchemas collects every source the project's dependencies +// contribute, adds the project's own apis/ directory, and runs one merged +// schema generation pass. This is the only way the dependency manager writes +// schemas: a per-source write would clobber the run-level files (root index, +// package manifest) of generators whose output describes the whole run, and +// leaves stale models for every language when dependencies are removed. +func (m *Manager) GenerateAllSchemas(ctx context.Context, ch async.EventChannel) error { + sources, err := m.CollectSources(ctx, ch) + if err != nil { + return err + } + + // The project's own apis/ directory is the second source group. A project + // declares it even when the directory does not exist yet (a Configuration + // with no XRDs of its own); the source tolerates a missing directory and + // simply contributes no resources, so creating it later is a source change + // the next pass regenerates from. + apis := smanager.NewProjectAPIsSource(m.projFS, m.proj.Spec.Paths) + + return m.schemas.GenerateFromMultipleSources(ctx, sources, apis) +} + +// addDependencyNoWrite fetches (or validates) a dependency without writing +// any schemas. For xpkg dependencies it fetches the package and walks its +// transitive dependencies, so callers get the package's GVK; git, HTTP and +// k8s sources need no fetching ahead of time -- the merged pass reads their +// resources through the sources CollectSources returns. func (m *Manager) addDependencyNoWrite(ctx context.Context, dep *v1alpha1.Dependency, refresh bool) (*schema.GroupVersionKind, error) { switch { case dep.Type == v1alpha1.DependencyTypeXpkg: @@ -376,16 +484,191 @@ func (m *Manager) addDependencyNoWrite(ctx context.Context, dep *v1alpha1.Depend return m.addPackage(ctx, xpkgRef(dep.Xpkg.Package, dep.Xpkg.Version), refresh) case dep.Git != nil: - return nil, m.schemas.Add(ctx, smanager.NewGitSource(*dep, m.gitCloner, m.gitAuthProvider)) + return nil, nil case dep.HTTP != nil: - return nil, m.schemas.Add(ctx, smanager.NewHTTPSource(*dep)) + return nil, nil case dep.K8s != nil: - return nil, m.schemas.Add(ctx, smanager.NewK8sSource(*dep)) + return nil, nil default: return nil, errors.New("dependency has no source configured") } } +// CollectSources returns all schema sources from the project's dependencies +// without generating schemas. This allows the caller to merge sources and +// generate schemas in a single pass, which the TypeScript generator requires: +// its output is one npm package per run, not per source, so generating +// dependencies one at a time would overwrite it on every source but the last. +func (m *Manager) CollectSources(ctx context.Context, ch async.EventChannel) ([]smanager.Source, error) { + m.resetVisited() + + eg, egCtx := errgroup.WithContext(ctx) + + // One slice per project dependency: a package contributes its own CRDs and + // those of everything its metadata depends on. Indexing by position keeps + // the flattened result in project-file order. + sourcesByIndex := make([][]smanager.Source, len(m.proj.Spec.Dependencies)) + + for i := range m.proj.Spec.Dependencies { + dep := &m.proj.Spec.Dependencies[i] + desc := "Updating dependency " + GetSourceDescription(*dep) + eg.Go(func() error { + ch.SendEvent(desc, async.EventStatusStarted) + srcs, err := m.collectSource(egCtx, dep) + if err != nil { + ch.SendEvent(desc, async.EventStatusFailure) + return err + } + ch.SendEvent(desc, async.EventStatusSuccess) + + sourcesByIndex[i] = srcs + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + + var sources []smanager.Source + for _, srcs := range sourcesByIndex { + sources = append(sources, srcs...) + } + + m.warnMultipleVersions() + + // Sort by ID. Project dependencies are collected concurrently and a + // transitive dependency shared by two of them lands under whichever + // goroutine claimed it first, so the flattened order is otherwise a race. + // The merged filesystem prefixes each source by its position, so a stable + // order keeps that tree reproducible between runs. + slices.SortFunc(sources, func(a, b smanager.Source) int { + return strings.Compare(a.ID(), b.ID()) + }) + + return sources, nil +} + +// recordCollected notes that source collection resolved repo to version (a +// ":tag" or "@digest" suffix). +func (m *Manager) recordCollected(repo, version string) { + m.visitedMu.Lock() + defer m.visitedMu.Unlock() + m.collected[repo] = append(m.collected[repo], version) +} + +// warnMultipleVersions warns about every package the project's dependencies +// resolved to more than one version. Both versions' CRDs go into the same +// generator run, so the kinds they share can get conflicting models; on main +// the copy from whichever source sorted last silently won. +func (m *Manager) warnMultipleVersions() { + m.visitedMu.Lock() + defer m.visitedMu.Unlock() + for _, repo := range slices.Sorted(maps.Keys(m.collected)) { + versions := slices.Sorted(slices.Values(m.collected[repo])) + if len(versions) < 2 { + continue + } + for i := range versions { + versions[i] = strings.TrimLeft(versions[i], ":@") + } + m.warnings = append(m.warnings, fmt.Sprintf("the project's dependencies resolve %s to more than one version (%s); schemas are generated from all of them, which can produce conflicting models for the kinds they share. Align the version constraints so they resolve to one version.", repo, strings.Join(versions, ", "))) + } +} + +// collectSource returns the schema source for a dependency without generating schemas. +func (m *Manager) collectSource(ctx context.Context, dep *v1alpha1.Dependency) ([]smanager.Source, error) { + desc := GetSourceDescription(*dep) + switch { + case dep.Type == v1alpha1.DependencyTypeXpkg: + if dep.Xpkg == nil || dep.Xpkg.Package == "" { + return nil, errors.Errorf("xpkg dependency %q is missing xpkg.package; set xpkg.package to a valid package reference", desc) + } + + // If the version is a digest, format the OCI ref as + // repo@digest. Otherwise, use repo:tag, where tag may be a semver + // constraint. + ref := dep.Xpkg.Package + if _, err := conregv1.NewHash(dep.Xpkg.Version); err == nil { + ref = fmt.Sprintf("%s@%s", ref, dep.Xpkg.Version) + } else if dep.Xpkg.Version != "" { + ref = fmt.Sprintf("%s:%s", ref, dep.Xpkg.Version) + } + + return m.collectPackageSource(ctx, ref) + case dep.Git != nil: + return []smanager.Source{smanager.NewGitSource(*dep, m.gitCloner, m.gitAuthProvider)}, nil + case dep.HTTP != nil: + return []smanager.Source{smanager.NewHTTPSource(*dep)}, nil + case dep.K8s != nil: + return []smanager.Source{smanager.NewK8sSource(*dep)}, nil + default: + return nil, errors.Errorf("dependency %q has no source configured; set exactly one of xpkg, git, http, or k8s", desc) + } +} + +// collectPackageSource fetches a package and returns its CRD source, followed +// by the sources of everything its metadata depends on. The merged schema pass +// generates from exactly what this returns, so a dependency missing here +// contributes no schemas. claim() gives cycle protection and dedupes diamonds. +func (m *Manager) collectPackageSource(ctx context.Context, ref string) ([]smanager.Source, error) { + if !m.claim(ref) { + // Already collected this invocation, directly or through another + // package's dependencies. + return nil, nil + } + + resolvedRef, version, err := m.resolver.Resolve(ctx, ref) + if err != nil { + return nil, errors.Wrapf(err, "cannot resolve package %q; check that the package exists and that the version or digest is valid", ref) + } + + // The claim above is on the reference as written, so one package named both + // by constraint and by exact version passes it twice and would be generated + // from twice. Claiming before the fetch also skips the dropped copy's + // download. + if !m.claim("resolved:" + resolvedRef.String()) { + return nil, nil + } + + pullPolicy := corev1.PullIfNotPresent + pkg, err := m.getPackage(ctx, resolvedRef.String(), pullPolicy) + if err != nil { + return nil, errors.Wrapf(err, "cannot download package %q; check registry access and credentials", ref) + } + + crdFS, err := clixpkg.CRDFilesystem(pkg.Package) + if err != nil { + return nil, errors.Wrapf(err, "cannot extract CRDs from package %q; check that it is a valid Crossplane package", ref) + } + + // Use the resolved version so constraint and exact-version inputs + // collapse to one schema-lock entry. + id := pkg.Source + "@" + pkg.Digest + if version != "" { + id = pkg.Source + ":" + version + } + + sources := []smanager.Source{smanager.NewXpkgSource(id, pkg.Digest, crdFS)} + m.recordCollected(pkg.Source, strings.TrimPrefix(id, pkg.Source)) + + // Depth first, in metadata order, so the result is deterministic for a + // given dependency graph. + for _, dep := range pkg.GetDependencies() { + repo := dependencyRepo(dep) + if repo == "" { + continue + } + depSources, err := m.collectPackageSource(ctx, xpkgRef(repo, dep.Version)) + if err != nil { + return nil, errors.Wrapf(err, "cannot collect transitive dependency %s of %s", repo, pkg.Source) + } + sources = append(sources, depSources...) + } + + return sources, nil +} + // Clean removes all generated schemas. func (m *Manager) Clean() error { return m.projFS.RemoveAll(m.proj.Spec.Paths.Schemas) diff --git a/internal/dependency/manager_test.go b/internal/dependency/manager_test.go index 0d2339d5..e4593b4c 100644 --- a/internal/dependency/manager_test.go +++ b/internal/dependency/manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package dependency import ( + "bytes" "context" "encoding/json" "fmt" @@ -71,46 +72,6 @@ spec: type: object ` -// configurationWithXRDPackageYAML is a Configuration package that bundles an -// XRD (rather than a raw CRD, like configurationPackageYAML above) - the -// shape that previously produced zero schemas. -const configurationWithXRDPackageYAML = `apiVersion: meta.pkg.crossplane.io/v1 -kind: Configuration -metadata: - name: example -spec: - crossplane: - version: ">=v1.14.0" ---- -apiVersion: apiextensions.crossplane.io/v1 -kind: CompositeResourceDefinition -metadata: - name: xdatabases.acme.example.com -spec: - group: acme.example.com - names: - kind: XDatabase - plural: xdatabases - singular: xdatabase - listKind: XDatabaseList - claimNames: - kind: Database - plural: databases - singular: database - listKind: DatabaseList - scope: LegacyCluster - versions: - - name: v1alpha1 - served: true - referenceable: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object -` - const providerPackageYAML = `apiVersion: meta.pkg.crossplane.io/v1 kind: Provider metadata: @@ -178,6 +139,13 @@ func parsedPackage(t *testing.T, body string) *parser.Package { return pkg } +// parsedTestPackage parses configurationPackageYAML once into a *parser.Package +// that the fake client can hand back from Get. +func parsedTestPackage(t *testing.T) *parser.Package { + t.Helper() + return parsedPackage(t, configurationPackageYAML) +} + // fakeClient is a fake xpkg.Client. Get returns a pre-canned Package per ref; // ListVersions returns the tags for the requested repo, falling back to a // fixed tag list, so a real Resolver can be wired on top. @@ -218,6 +186,16 @@ func (f *fakeClient) getCount(ref string) int { return f.gets[ref] } +func makePackage(t *testing.T, source, digest, version string) *runtimexpkg.Package { + t.Helper() + return &runtimexpkg.Package{ + Package: parsedTestPackage(t), + Source: source, + Digest: digest, + Version: version, + } +} + func makePackageWithBody(t *testing.T, source, digest, version, body string) *runtimexpkg.Package { t.Helper() return &runtimexpkg.Package{ @@ -228,18 +206,22 @@ func makePackageWithBody(t *testing.T, source, digest, version, body string) *ru } } -func newTestManager(t *testing.T, fc *fakeClient, generators ...generator.Interface) (*Manager, afero.Fs) { +// newTestManagerWithDeps builds a manager for a project with the given +// dependencies declared, as `crossplane dependency add` leaves it. Pass no +// deps for a dependency-less project. +func newTestManagerWithDeps(t *testing.T, fc *fakeClient, deps ...v1alpha1.Dependency) (*Manager, afero.Fs) { t.Helper() schemaFS := afero.NewMemMapFs() m := NewManager( &v1alpha1.Project{ Spec: v1alpha1.ProjectSpec{ - Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + Dependencies: deps, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas", APIs: "apis"}, }, }, afero.NewMemMapFs(), WithSchemaFS(schemaFS), - WithSchemaGenerators(generators), + WithSchemaGenerators([]generator.Interface{}), WithXpkgClient(fc), WithResolver(clixpkg.NewResolver(fc)), ) @@ -506,6 +488,18 @@ func TestManager_AddDependency(t *testing.T) { if tc.fetchAt != "" { fc.packages[tc.fetchAt] = makePackageWithBody(t, tc.dep.Xpkg.Package, digestVal, "", tc.pkg.body) } + // GenerateAllSchemas collects sources for every declared + // dependency, so any deps the project starts with must be + // servable too. + for _, existing := range tc.inputDeps { + if existing.Xpkg == nil || existing.Xpkg.Package == tc.dep.Xpkg.Package { + continue + } + ref := existing.Xpkg.Package + ":" + existing.Xpkg.Version + if _, ok := fc.packages[ref]; !ok { + fc.packages[ref] = makePackageWithBody(t, existing.Xpkg.Package, digestVal, "", fnInfo.body) + } + } projFS := afero.NewMemMapFs() proj := &v1alpha1.Project{ @@ -567,104 +561,56 @@ func TestManager_AddDependency(t *testing.T) { } } -func TestManager_AddPackage(t *testing.T) { - const ( - cfgXRDPkg = "xpkg.crossplane.io/crossplane-contrib/configuration-xrd" - cfgXRDTag = "v0.1.0" - ) - +func TestManager_CollectSources_RefNormalizes(t *testing.T) { + // The source ID is what the lock file records, so a package named by + // constraint, by exact tag, or by digest must still collapse to one + // canonical xpkg:// ID in each spelling. tests := map[string]struct { - ref string + dep *v1alpha1.Dependency tags []string fetchAt string - // body is the package YAML served by the fake client; empty - // defaults to configurationPackageYAML. - body string - // generators are the schema generators wired into the manager; - // empty/nil means no schema files are actually rendered. - generators []generator.Interface - wantKey string - // wantSchemaGlob, when set, must match at least one file in - // schemaFS after AddPackage. - wantSchemaGlob string + wantID string }{ "ConstraintCollapsesToResolvedVersion": { - ref: "pkg.example/foo:>=v0.0.0", + dep: xpkgDep("pkg.example/foo", ">=v0.0.0"), tags: []string{"v0.5.2"}, fetchAt: "pkg.example/foo:v0.5.2", - wantKey: "xpkg://pkg.example/foo:v0.5.2", + wantID: "xpkg://pkg.example/foo:v0.5.2", }, "ExactVersionMatchesResolved": { - ref: "pkg.example/foo:v0.5.2", + dep: xpkgDep("pkg.example/foo", "v0.5.2"), tags: []string{"v0.5.2"}, fetchAt: "pkg.example/foo:v0.5.2", - wantKey: "xpkg://pkg.example/foo:v0.5.2", + wantID: "xpkg://pkg.example/foo:v0.5.2", }, "DigestRefUsesDigestForm": { - ref: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + dep: xpkgDep("pkg.example/foo", "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"), fetchAt: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", - wantKey: "xpkg://pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", - }, - // ConfigurationWithXRDGeneratesSchemas verifies that adding a - // Configuration dependency that bundles XRDs (rather than raw - // CRDs) generates real schema output, not just an empty - // successful pass. Before internal/xpkg.CRDFilesystem learned to - // convert XRDs to their derived CRD form, this produced a lock - // entry but zero schema content. - "ConfigurationWithXRDGeneratesSchemas": { - ref: cfgXRDPkg + ":" + cfgXRDTag, - tags: []string{cfgXRDTag}, - fetchAt: cfgXRDPkg + ":" + cfgXRDTag, - body: configurationWithXRDPackageYAML, - generators: generator.Filter(generator.AllLanguages(), []string{v1alpha1.SchemaLanguageJSON}), - wantKey: "xpkg://" + cfgXRDPkg + ":" + cfgXRDTag, - wantSchemaGlob: "json/*.schema.json", + wantID: "xpkg://pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - body := tc.body - if body == "" { - body = configurationPackageYAML - } fc := &fakeClient{ packages: map[string]*runtimexpkg.Package{ - tc.fetchAt: makePackageWithBody(t, refRepo(tc.ref), "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", "", body), + tc.fetchAt: makePackage(t, "pkg.example/foo", "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", ""), }, tags: tc.tags, } - m, schemaFS := newTestManager(t, fc, tc.generators...) + m, _ := newTestManagerWithDeps(t, fc, *tc.dep) - if _, err := m.AddPackage(context.Background(), tc.ref, false); err != nil { - t.Fatalf("AddPackage: %v", err) - } - - bs, err := afero.ReadFile(schemaFS, ".lock.json") + sources, err := m.CollectSources(context.Background(), nil) if err != nil { - t.Fatalf("read lock: %v", err) - } - var got struct { - Packages map[string]string `json:"packages"` + t.Fatalf("CollectSources: %v", err) } - if err := json.Unmarshal(bs, &got); err != nil { - t.Fatalf("unmarshal lock: %v", err) + ids := make([]string, 0, len(sources)) + for _, src := range sources { + ids = append(ids, src.ID()) } - if _, ok := got.Packages[tc.wantKey]; !ok { - t.Errorf("lock has no entry for %q; keys = %v", tc.wantKey, slices.Collect(maps.Keys(got.Packages))) - } - if len(got.Packages) != 1 { - t.Errorf("lock packages = %d, want 1; got %v", len(got.Packages), got.Packages) - } - - if tc.wantSchemaGlob != "" { - files, err := afero.Glob(schemaFS, tc.wantSchemaGlob) - if err != nil { - t.Fatalf("glob generated schemas: %v", err) - } - if len(files) == 0 { - t.Errorf("no files matched %q; no schemas were generated", tc.wantSchemaGlob) - } + want := []string{tc.wantID} + if diff := cmp.Diff(want, ids); diff != "" { + t.Errorf("source IDs (-want +got):\n%s", diff) } }) } @@ -682,7 +628,10 @@ func refRepo(ref string) string { return ref } -// readLockKeys returns the package keys recorded in the schema lock file. +// readLockKeys returns the dependency-source keys recorded in the schema +// lock file. Source entries live under groups., one map per source +// group; the apis entry (fs://apis) belongs to a different group, so only the +// dependency keys are returned. func readLockKeys(t *testing.T, schemaFS afero.Fs) []string { t.Helper() bs, err := afero.ReadFile(schemaFS, ".lock.json") @@ -690,17 +639,19 @@ func readLockKeys(t *testing.T, schemaFS afero.Fs) []string { t.Fatalf("read lock: %v", err) } var lock struct { - Packages map[string]string `json:"packages"` + Groups map[string]map[string]string `json:"groups"` } if err := json.Unmarshal(bs, &lock); err != nil { t.Fatalf("unmarshal lock: %v", err) } - keys := slices.Collect(maps.Keys(lock.Packages)) + keys := slices.Collect(maps.Keys(lock.Groups["dependencies"])) slices.Sort(keys) return keys } -func TestManager_AddPackage_TransitiveDeps(t *testing.T) { +func TestManager_CollectSources_TransitiveWalk(t *testing.T) { + // What CollectSources returns is what the merged schema pass generates + // from, so a transitive dependency missing here means missing schemas. const ( provA = "xpkg.example/prov-a" provB = "xpkg.example/prov-b" @@ -716,7 +667,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { tests := map[string]struct { packages []pkgFixture tagsByRepo map[string][]string - addRefs []string + declaredDeps []string wantLockKeys []string wantErr string wantGets map[string]int @@ -730,7 +681,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provA: {"v0.1.0"}, family: {"v1.0.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, + declaredDeps: []string{provA + ":v0.1.0"}, wantLockKeys: []string{ "xpkg://" + family + ":v1.0.0", "xpkg://" + provA + ":v0.1.0", @@ -745,7 +696,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provA: {"v0.1.0"}, family: {"v1.2.3"}, }, - addRefs: []string{provA + ":v0.1.0"}, + declaredDeps: []string{provA + ":v0.1.0"}, wantLockKeys: []string{ "xpkg://" + family + ":v1.2.3", "xpkg://" + provA + ":v0.1.0", @@ -759,7 +710,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { tagsByRepo: map[string][]string{ provA: {"v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, + declaredDeps: []string{provA + ":v0.1.0"}, wantLockKeys: []string{ "xpkg://" + family + "@" + digestVal, "xpkg://" + provA + ":v0.1.0", @@ -774,7 +725,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provA: {"v0.1.0"}, family: {"v1.0.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, + declaredDeps: []string{provA + ":v0.1.0"}, wantLockKeys: []string{ "xpkg://" + family + ":v1.0.0", "xpkg://" + provA + ":v0.1.0", @@ -791,7 +742,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provB: {"v0.2.0"}, family: {"v1.0.0"}, }, - addRefs: []string{provA + ":v0.1.0", provB + ":v0.2.0"}, + declaredDeps: []string{provA + ":v0.1.0", provB + ":v0.2.0"}, wantLockKeys: []string{ "xpkg://" + family + ":v1.0.0", "xpkg://" + provA + ":v0.1.0", @@ -810,7 +761,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provA: {"v0.1.0"}, provB: {"v0.2.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, + declaredDeps: []string{provA + ":v0.1.0"}, wantLockKeys: []string{ "xpkg://" + provA + ":v0.1.0", "xpkg://" + provB + ":v0.2.0", @@ -824,11 +775,10 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { provA: {"v0.1.0"}, family: {}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantErr: family + ":v9.9.9", + declaredDeps: []string{provA + ":v0.1.0"}, + wantErr: family + ":v9.9.9", }, } - for name, tc := range tests { t.Run(name, func(t *testing.T) { fc := &fakeClient{ @@ -838,14 +788,14 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { for _, p := range tc.packages { fc.packages[p.ref] = makePackageWithBody(t, refRepo(p.ref), "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", "", p.body) } - m, schemaFS := newTestManager(t, fc) - - var err error - for _, ref := range tc.addRefs { - if _, err = m.AddPackage(context.Background(), ref, false); err != nil { - break - } + deps := make([]v1alpha1.Dependency, len(tc.declaredDeps)) + for i, ref := range tc.declaredDeps { + repo, tag, _ := strings.Cut(ref, ":") + deps[i] = *xpkgDep(repo, tag) } + m, schemaFS := newTestManagerWithDeps(t, fc, deps...) + + err := m.GenerateAllSchemas(context.Background(), nil) if tc.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { t.Fatalf("expected error containing %q, got %v", tc.wantErr, err) @@ -853,7 +803,7 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { return } if err != nil { - t.Fatalf("AddPackage: %v", err) + t.Fatalf("GenerateAllSchemas: %v", err) } want := slices.Clone(tc.wantLockKeys) @@ -871,7 +821,197 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { } } -func TestManager_AddAll_SharedTransitiveDep(t *testing.T) { +func TestManager_CollectSources_Transitive(t *testing.T) { + // The merged schema pass generates from exactly what CollectSources + // returns, so a transitive dependency missing here contributes no schemas. + // provA and provB both depend on family, which must appear once. + const ( + provA = "xpkg.example/prov-a" + provB = "xpkg.example/prov-b" + family = "xpkg.example/family" + digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + provB + ":v0.2.0": makePackageWithBody(t, provB, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + provB: {"v0.2.0"}, + family: {"v1.0.0"}, + }, + } + + m := NewManager( + &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Dependencies: []v1alpha1.Dependency{ + *xpkgDep(provA, "v0.1.0"), + *xpkgDep(provB, "v0.2.0"), + }, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + }, + afero.NewMemMapFs(), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + var ch async.EventChannel // nil channel; SendEvent is a no-op. + sources, err := m.CollectSources(context.Background(), ch) + if err != nil { + t.Fatalf("CollectSources: %v", err) + } + + got := make([]string, 0, len(sources)) + for _, src := range sources { + got = append(got, src.ID()) + } + + // Sorted by ID, and family only once even though both providers depend on + // it. Sorting is what makes this assertable: the project dependencies are + // collected concurrently, so a shared transitive dependency would otherwise + // land under whichever goroutine claimed it first. + want := []string{ + "xpkg://" + family + ":v1.0.0", + "xpkg://" + provA + ":v0.1.0", + "xpkg://" + provB + ":v0.2.0", + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("source IDs (-want +got):\n%s", diff) + } + + if got := fc.getCount(family + ":v1.0.0"); got != 1 { + t.Errorf("fetch count for %s = %d, want 1", family+":v1.0.0", got) + } +} + +// A second top-level call on the same Manager must traverse every dependency +// again, not silently skip the ones the first call already claimed. Without +// resetVisited, a caller that retries CollectSources after a transient +// failure, or simply calls it twice, would see progressively fewer sources. +func TestManager_CollectSources_ReusableAcrossCalls(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", providerPackageYAML), + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + }, + } + + m := NewManager( + &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Dependencies: []v1alpha1.Dependency{ + *xpkgDep(provA, "v0.1.0"), + }, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + }, + afero.NewMemMapFs(), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + var ch async.EventChannel // nil channel; SendEvent is a no-op. + want := []string{"xpkg://" + provA + ":v0.1.0"} + + for _, label := range []string{"first call", "second call"} { + sources, err := m.CollectSources(context.Background(), ch) + if err != nil { + t.Fatalf("CollectSources (%s): %v", label, err) + } + + got := make([]string, 0, len(sources)) + for _, src := range sources { + got = append(got, src.ID()) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("source IDs on %s (-want +got):\n%s", label, diff) + } + } +} + +func TestManager_CollectSources_RangeAndExactCollapse(t *testing.T) { + // A package reached once through a constraint and once through an exact + // version is still one package. claim() records the reference as written, + // before Resolve canonicalizes it, so both spellings pass it and produce + // two sources with the same ID - which makes the merged pass generate from + // the same CRDs twice. + const ( + provA = "xpkg.example/prov-a" + family = "xpkg.example/family" + digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {"v1.0.0"}, + }, + } + + m := NewManager( + &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Dependencies: []v1alpha1.Dependency{ + // Depends on family:v1.0.0 through its metadata. + *xpkgDep(provA, "v0.1.0"), + // And the project names the same package by constraint. + *xpkgDep(family, ">=v1.0.0"), + }, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + }, + afero.NewMemMapFs(), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + var ch async.EventChannel // nil channel; SendEvent is a no-op. + sources, err := m.CollectSources(context.Background(), ch) + if err != nil { + t.Fatalf("CollectSources: %v", err) + } + + got := make([]string, 0, len(sources)) + for _, src := range sources { + got = append(got, src.ID()) + } + + want := []string{ + "xpkg://" + family + ":v1.0.0", + "xpkg://" + provA + ":v0.1.0", + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("source IDs (-want +got):\n%s", diff) + } + + if got := fc.getCount(family + ":v1.0.0"); got != 1 { + t.Errorf("fetch count for %s = %d, want 1", family+":v1.0.0", got) + } +} + +func TestManager_GenerateAllSchemas_SharedTransitiveDep(t *testing.T) { const ( provA = "xpkg.example/prov-a" provB = "xpkg.example/prov-b" @@ -911,8 +1051,8 @@ func TestManager_AddAll_SharedTransitiveDep(t *testing.T) { ) var ch async.EventChannel // nil channel; SendEvent is a no-op. - if err := m.AddAll(context.Background(), ch); err != nil { - t.Fatalf("AddAll: %v", err) + if err := m.GenerateAllSchemas(context.Background(), ch); err != nil { + t.Fatalf("GenerateAllSchemas: %v", err) } want := []string{ @@ -1013,3 +1153,148 @@ func TestManager_AddDependency_TransitiveNotPersisted(t *testing.T) { t.Errorf("on-disk project deps (-want +got):\n%s", diff) } } + +// A dependency whose schemas cannot be generated must not be written to the +// project file. Git, HTTP and k8s sources are only fetched by the generation +// pass, so persisting first would record a dependency that was never checked. +func TestManager_AddDependency_FailedGenerationNotPersisted(t *testing.T) { + projFS := afero.NewMemMapFs() + proj := &v1alpha1.Project{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "dev.crossplane.io/v1alpha1", + Kind: "Project", + }, + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: v1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/test/test", + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + } + bs, err := yaml.Marshal(proj) + if err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "crossplane-project.yaml", bs, 0o644); err != nil { + t.Fatal(err) + } + + m := NewManager(proj, projFS, + WithProjectFile("crossplane-project.yaml"), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + ) + + // Nothing listens on port 1, so fetching this source fails. + dep := &v1alpha1.Dependency{ + Type: v1alpha1.DependencyTypeCRD, + HTTP: &v1alpha1.HTTPDependency{URL: "http://127.0.0.1:1/crds.yaml"}, + } + if err := m.AddDependency(context.Background(), dep); err == nil { + t.Fatal("AddDependency succeeded for an unreachable HTTP source") + } + + if len(proj.Spec.Dependencies) != 0 { + t.Errorf("in-memory project dependencies = %v, want none", proj.Spec.Dependencies) + } + gotBytes, err := afero.ReadFile(projFS, "crossplane-project.yaml") + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(string(bs), string(gotBytes)); diff != "" { + t.Errorf("project file changed (-want +got):\n%s", diff) + } +} + +// Two dependencies that pull in one package at different versions get a +// warning naming the package and both versions: both versions' CRDs go into +// the same generator run. +func TestManager_CollectSources_WarnsOnMultipleVersions(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + provB = "xpkg.example/prov-b" + family = "xpkg.example/family" + digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + provB + ":v0.1.0": makePackageWithBody(t, provB, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.1.0")), + family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), + family + ":v1.1.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + provB: {"v0.1.0"}, + family: {"v1.0.0", "v1.1.0"}, + }, + } + + proj := &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + Dependencies: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(provB, "v0.1.0")}, + }, + } + + var warnings bytes.Buffer + m := NewManager(proj, afero.NewMemMapFs(), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + if _, err := m.CollectSources(context.Background(), nil); err != nil { + t.Fatalf("CollectSources: %v", err) + } + m.PrintWarnings(&warnings) + want := "warning: the project's dependencies resolve " + family + " to more than one version (v1.0.0, v1.1.0)" + if !strings.Contains(warnings.String(), want) { + t.Errorf("warnings = %q, want it to contain %q", warnings.String(), want) + } + + // A second collection over the same project warns again, once: the + // record of collected versions is per traversal. + warnings.Reset() + if _, err := m.CollectSources(context.Background(), nil); err != nil { + t.Fatalf("CollectSources: %v", err) + } + m.PrintWarnings(&warnings) + if got := strings.Count(warnings.String(), "warning:"); got != 1 { + t.Errorf("second collection printed %d warnings, want 1:\n%s", got, warnings.String()) + } +} + +// One version of each package collects without warnings. +func TestManager_CollectSources_NoWarningForOneVersion(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + ) + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", providerPackageYAML)}, + tagsByRepo: map[string][]string{provA: {"v0.1.0"}}, + } + proj := &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + Dependencies: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(provA, ">=v0.1.0")}, + }, + } + + var warnings bytes.Buffer + m := NewManager(proj, afero.NewMemMapFs(), + WithSchemaFS(afero.NewMemMapFs()), + WithSchemaGenerators([]generator.Interface{}), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + if _, err := m.CollectSources(context.Background(), nil); err != nil { + t.Fatalf("CollectSources: %v", err) + } + m.PrintWarnings(&warnings) + if warnings.Len() != 0 { + t.Errorf("unexpected warnings: %s", warnings.String()) + } +} diff --git a/internal/project/build.go b/internal/project/build.go index 81de2ca7..6d84c912 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -262,19 +262,22 @@ func (b *Builder) Build(ctx context.Context, project *devv1alpha1.Project, proje } o.eventCh.SendEvent("Collecting resources", async.EventStatusSuccess) - // Generate schemas for declared dependencies. The dependency manager - // short-circuits sources whose recorded version still matches, so this is - // cheap on the steady-state path. + // Collect all schema sources (dependencies + local APIs) and generate + // schemas in a single pass, so the generated package's root index covers + // every group rather than only the source that ran last. + var depSources []manager.Source if b.dependencyManager != nil { - if err := b.dependencyManager.AddAll(ctx, o.eventCh); err != nil { - return nil, errors.Wrap(err, "failed to generate dependency schemas") + var err error + depSources, err = b.dependencyManager.CollectSources(ctx, o.eventCh) + if err != nil { + return nil, errors.Wrap(err, "cannot load schemas from project dependencies; check that each dependency is reachable and contains valid API definitions") } } - - // Generate language-specific schemas from XRDs. if b.schemaManager != nil { + apis := manager.NewProjectAPIsSource(projectFS, project.Spec.Paths) + o.eventCh.SendEvent("Generating schemas", async.EventStatusStarted) - if _, err := b.schemaManager.Generate(ctx, manager.NewFSSource(project.Spec.Paths.APIs, apisSource)); err != nil { + if err := b.schemaManager.GenerateFromMultipleSources(ctx, depSources, apis); err != nil { o.eventCh.SendEvent("Generating schemas", async.EventStatusFailure) return nil, errors.Wrap(err, "failed to generate schemas") } diff --git a/internal/schemas/generator/go.go b/internal/schemas/generator/go.go index 93b0ae4d..95b1fa2d 100644 --- a/internal/schemas/generator/go.go +++ b/internal/schemas/generator/go.go @@ -240,6 +240,13 @@ func (goGenerator) Language() string { return devv1alpha1.SchemaLanguageGo } +// Fingerprint describes the output-affecting options of this generator: the +// accessor and runtime-object emission settings. The generation logic itself +// changes only with the CLI version, which the lock fingerprints separately. +func (g goGenerator) Fingerprint() string { + return fmt.Sprintf("accessors=%t,runtimeobjects=%t", g.accessors, g.runtimeObjects) +} + // GenerateFromCRD generates Go schemas for the CRDs in the given filesystem. func (g goGenerator) GenerateFromCRD(_ context.Context, fromFS afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { openAPIs, err := collectCRDOpenAPIs(fromFS) @@ -275,8 +282,9 @@ func (g goGenerator) GenerateFromCRD(_ context.Context, fromFS afero.Fs, _ runne } } - // Generate separate files for each K8s package - for pkg, schemas := range k8sSchemasByPackage { + // Generate separate files for each K8s package, in a stable order. + for _, pkg := range slices.Sorted(maps.Keys(k8sSchemasByPackage)) { + schemas := k8sSchemasByPackage[pkg] if err := g.generateSharedK8sPackage(schemaFS, pkg, schemas); err != nil { return nil, err } @@ -1199,8 +1207,9 @@ func generateK8sSharedSchemas(openAPISpecs []*spec3.OpenAPI, schemaFS afero.Fs, } } - // Generate separate files for each K8s package - for pkg, schemas := range k8sSchemasByPackage { + // Generate separate files for each K8s package, in a stable order. + for _, pkg := range slices.Sorted(maps.Keys(k8sSchemasByPackage)) { + schemas := k8sSchemasByPackage[pkg] if len(schemas) == 0 { continue } @@ -1304,7 +1313,14 @@ func generateModelsWithGVK(openAPISpecs []*spec3.OpenAPI, schemaFS afero.Fs, g g for _, openAPISpec := range openAPISpecs { gvkGroups := groupSchemasByGVK(openAPISpec) - for gvkKey, schemas := range gvkGroups { + // Sorted: every group's spec shares the same schema pointers, and the + // mutators in generateGVKGroupCode edit them in place, so what a + // group generates depends on the groups processed before it. In map + // order that varied run to run (TokenRequest's enum constants in + // authentication/v1 were sometimes prefixed with the type name and + // sometimes not). + for _, gvkKey := range slices.Sorted(maps.Keys(gvkGroups)) { + schemas := gvkGroups[gvkKey] // Skip groups whose schemas are all shared k8s package schemas // (e.g. autoscaling/v1's Scale). They're generated by // generateK8sSharedSchemas, and goRemoveK8s would leave this group diff --git a/internal/schemas/generator/go_test.go b/internal/schemas/generator/go_test.go index 358f84a1..b9c218e8 100644 --- a/internal/schemas/generator/go_test.go +++ b/internal/schemas/generator/go_test.go @@ -21,6 +21,7 @@ import ( "go/ast" "go/parser" "go/token" + "io/fs" "path/filepath" "strings" "testing" @@ -307,3 +308,38 @@ func TestGenerateFromOpenAPIGo(t *testing.T) { } } } + +// The same OpenAPI input must produce the same bytes on every run: the merged +// schema pass compares shared files byte for byte, and users' code depends on +// the generated constant names. GVK groups share schema pointers that the +// generator mutates in place, so processing them in map order once made enum +// constant names in authentication/v1 differ between runs. +func TestGenerateFromOpenAPIGoDeterministic(t *testing.T) { + inputFS := afero.NewBasePathFs(afero.FromIOFS{FS: testdataJSONFS}, "testdata") + + render := func() map[string]string { + schemaFS, err := goGenerator{}.GenerateFromOpenAPI(t.Context(), inputFS, nil) + if err != nil { + t.Fatal(err) + } + files := map[string]string{} + if err := afero.Walk(schemaFS, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + bs, err := afero.ReadFile(schemaFS, p) + files[p] = string(bs) + return err + }); err != nil { + t.Fatal(err) + } + return files + } + + first := render() + for i := range 5 { + if diff := cmp.Diff(first, render()); diff != "" { + t.Fatalf("run %d differs from the first (-first +run):\n%s", i+2, diff) + } + } +} diff --git a/internal/schemas/generator/interface.go b/internal/schemas/generator/interface.go index 3ca4a2a0..264d267d 100644 --- a/internal/schemas/generator/interface.go +++ b/internal/schemas/generator/interface.go @@ -32,6 +32,13 @@ type Interface interface { Language() string GenerateFromCRD(ctx context.Context, fs afero.Fs, runner runner.SchemaRunner) (afero.Fs, error) GenerateFromOpenAPI(ctx context.Context, fs afero.Fs, runner runner.SchemaRunner) (afero.Fs, error) + + // Fingerprint returns a stable string describing everything this + // generator configures that determines its output: its pinned toolchain + // image, and any options that change what it emits. Two fingerprints that + // differ mean the same sources generate different files, so the schema + // manager treats cached output made under the old one as stale. + Fingerprint() string } // options holds configurable behavior shared across generators. diff --git a/internal/schemas/generator/json.go b/internal/schemas/generator/json.go index 0c685139..83962db2 100644 --- a/internal/schemas/generator/json.go +++ b/internal/schemas/generator/json.go @@ -54,6 +54,13 @@ func (jsonGenerator) Language() string { return devv1alpha1.SchemaLanguageJSON } +// Fingerprint reports that this generator has no output-affecting options. +// Its output changes only with the CLI version, which the lock fingerprints +// separately. +func (jsonGenerator) Fingerprint() string { + return "" +} + // GenerateFromCRD generates jsonschemas for the CRDs in the given filesystem. func (jsonGenerator) GenerateFromCRD(_ context.Context, fromFS afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { openAPIs, err := collectCRDOpenAPIs(fromFS) diff --git a/internal/schemas/generator/kcl.go b/internal/schemas/generator/kcl.go index 0a3e9cff..13e293e0 100644 --- a/internal/schemas/generator/kcl.go +++ b/internal/schemas/generator/kcl.go @@ -52,6 +52,12 @@ const ( kclModelsFolder = "models" kclAdoptModelsStructure = "sorted" kclImage = "docker.io/kcllang/kcl:v0.11.2" + // kclEdition is the kcl.mod edition the pinned kclImage's `kcl import` + // writes. The OpenAPI path writes kcl.mod itself (the import tool only + // understands CRDs); it must match what the CRD path's import produces, + // or a project with both CRD and OpenAPI sources reports the two + // kcl.mod files as a conflict. + kclEdition = "v0.11.2" ) type kclGenerator struct{} @@ -60,6 +66,12 @@ func (kclGenerator) Language() string { return devv1alpha1.SchemaLanguageKCL } +// Fingerprint describes the output-affecting configuration of this generator: +// the pinned KCL toolchain image it runs generation in. +func (kclGenerator) Fingerprint() string { + return "image=" + kclImage +} + // GenerateFromCRD generates KCL schema files from the XRDs and CRDs fromFS. func (kclGenerator) GenerateFromCRD(ctx context.Context, fromFS afero.Fs, generator runner.SchemaRunner) (afero.Fs, error) { //nolint:gocognit // generate kcl schemas crdFS := afero.NewMemMapFs() @@ -321,11 +333,11 @@ func (kclGenerator) GenerateFromOpenAPI(_ context.Context, fromFS afero.Fs, _ ru } } - kclModContent := `[package] + kclModContent := fmt.Sprintf(`[package] name = "models" -edition = "v0.10.0" +edition = %q version = "0.0.1" -` +`, kclEdition) if err := afero.WriteFile(schemaFS, filepath.Join(kclModelsFolder, "kcl.mod"), []byte(kclModContent), 0o644); err != nil { return nil, errors.Wrap(err, "failed to write kcl.mod") } @@ -761,7 +773,11 @@ This file was generated by crossplane. DO NOT EDIT. } } - for imp := range imports { + // Sorted: import order must not depend on map iteration, or two + // generation passes over the same schema can produce different bytes + // for the same file. + sortedImports := slices.Sorted(maps.Keys(imports)) + for _, imp := range sortedImports { alias := getImportAlias(imp) sb.WriteString("import " + imp + " as " + alias + "\n") } diff --git a/internal/schemas/generator/kcl_test.go b/internal/schemas/generator/kcl_test.go index a6a40f25..bdaf71c6 100644 --- a/internal/schemas/generator/kcl_test.go +++ b/internal/schemas/generator/kcl_test.go @@ -16,7 +16,14 @@ limitations under the License. package generator -import "testing" +import ( + "fmt" + "slices" + "strings" + "testing" + + "k8s.io/kube-openapi/pkg/validation/spec" +) func TestIsAPIVersion(t *testing.T) { t.Parallel() @@ -47,3 +54,56 @@ func TestIsAPIVersion(t *testing.T) { }) } } + +// The OpenAPI path writes kcl.mod itself, and the CRD path gets it from +// `kcl import` in kclImage. The two must agree on the edition, or a project +// with both CRD and OpenAPI sources reports their kcl.mod files as a +// conflict. +func TestKCLEditionMatchesImage(t *testing.T) { + t.Parallel() + + if !strings.HasSuffix(kclImage, ":"+kclEdition) { + t.Errorf("kclEdition %q does not match the tag of kclImage %q", kclEdition, kclImage) + } +} + +// Imports must come out in the same order on every run: the merged pass +// compares shared files byte for byte, and Go map order is random. +func TestGenerateKCLFileImportsSorted(t *testing.T) { + t.Parallel() + + refs := []string{ + "io.k8s.api.core.v1.PodSpec", + "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta", + "io.k8s.api.apps.v1.DeploymentStrategy", + "io.k8s.api.batch.v1.JobSpec", + "io.k8s.api.networking.v1.IngressSpec", + } + schema := &spec.Schema{SchemaProps: spec.SchemaProps{Properties: map[string]spec.Schema{}}} + for i, ref := range refs { + schema.Properties[fmt.Sprintf("p%d", i)] = spec.Schema{ + SchemaProps: spec.SchemaProps{Ref: spec.MustCreateRef("#/components/schemas/" + ref)}, + } + } + + first := generateKCLFile("io.example.v1.Widget", schema, map[string]*spec.Schema{}) + + var imports []string + for line := range strings.SplitSeq(first, "\n") { + if strings.HasPrefix(line, "import ") { + imports = append(imports, line) + } + } + if len(imports) != len(refs) { + t.Fatalf("got %d imports, want %d:\n%s", len(imports), len(refs), first) + } + if !slices.IsSorted(imports) { + t.Errorf("imports are not sorted:\n%s", strings.Join(imports, "\n")) + } + + for range 20 { + if got := generateKCLFile("io.example.v1.Widget", schema, map[string]*spec.Schema{}); got != first { + t.Fatalf("generateKCLFile output differs between runs:\n%s\nvs\n%s", first, got) + } + } +} diff --git a/internal/schemas/generator/python.go b/internal/schemas/generator/python.go index b6e84617..84f126fb 100644 --- a/internal/schemas/generator/python.go +++ b/internal/schemas/generator/python.go @@ -59,6 +59,12 @@ func (pythonGenerator) Language() string { return devv1alpha1.SchemaLanguagePython } +// Fingerprint describes the output-affecting configuration of this generator: +// the pinned datamodel-code-generator image it runs generation in. +func (pythonGenerator) Fingerprint() string { + return "image=" + pythonImage +} + // GenerateFromCRD generates Python schema files from the XRDs and CRDs fromFS. func (p pythonGenerator) GenerateFromCRD(ctx context.Context, fromFS afero.Fs, generator runner.SchemaRunner) (afero.Fs, error) { //nolint:gocognit // generation of schemas for python crdFS := afero.NewMemMapFs() @@ -758,7 +764,57 @@ packages = ["models"] // that lacks one and writing a pyproject.toml at the schemas root. func finalizePythonSchemas(fsys afero.Fs) error { pkgRoot := filepath.Join(pythonModelsFolder, pythonPackageRoot) - if err := afero.Walk(fsys, pkgRoot, func(path string, info os.FileInfo, err error) error { + + // Strip provenance before anything else compares file bytes. + if err := stripCodegenFilenameHeaders(fsys, pkgRoot); err != nil { + return err + } + + if err := ensurePythonInitFiles(fsys, pkgRoot); err != nil { + return err + } + + return afero.WriteFile(fsys, filepath.Join(pythonModelsFolder, "pyproject.toml"), []byte(pythonSchemasPyproject), 0o644) +} + +// stripCodegenFilenameHeaders removes the "# filename: " line +// datamodel-code-generator writes under its "generated by" comment. The line +// records which input file contributed the model, so otherwise-identical +// copies of a shared model -- one wrapped by a dependency's CRD, one by the +// project's own XRD (apimachinery's ObjectMeta is the usual case) -- differ +// only in it. Model files that meet in the assembled tree must be +// byte-identical, so the provenance line has to go. +func stripCodegenFilenameHeaders(fsys afero.Fs, pkgRoot string) error { + return afero.Walk(fsys, pkgRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || filepath.Ext(path) != ".py" { + return nil + } + bs, err := afero.ReadFile(fsys, path) + if err != nil { + return errors.Wrapf(err, "reading %s", path) + } + lines := strings.Split(string(bs), "\n") + for i, line := range lines { + if strings.HasPrefix(line, "# filename:") { + without := make([]string, 0, len(lines)-1) + without = append(without, lines[:i]...) + without = append(without, lines[i+1:]...) + return afero.WriteFile(fsys, path, []byte(strings.Join(without, "\n")), 0o644) + } + if i > 5 { + // The header, when present, is at the very top of the file. + break + } + } + return nil + }) +} + +func ensurePythonInitFiles(fsys afero.Fs, pkgRoot string) error { + return afero.Walk(fsys, pkgRoot, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -772,9 +828,5 @@ func finalizePythonSchemas(fsys afero.Fs) error { } } return nil - }); err != nil { - return err - } - - return afero.WriteFile(fsys, filepath.Join(pythonModelsFolder, "pyproject.toml"), []byte(pythonSchemasPyproject), 0o644) + }) } diff --git a/internal/schemas/generator/python_test.go b/internal/schemas/generator/python_test.go index edb9e9b3..fb3ad940 100644 --- a/internal/schemas/generator/python_test.go +++ b/internal/schemas/generator/python_test.go @@ -155,3 +155,42 @@ func TestAdjustLeadingDots(t *testing.T) { }) } } + +// datamodel-code-generator records the input file in each model's header. +// Two copies of a shared model then differ only in that line, so it is +// removed; the rest of the file, and files without the header, are kept. +func TestStripCodegenFilenameHeaders(t *testing.T) { + fsys := afero.NewMemMapFs() + withHeader := "# generated by datamodel-codegen:\n# filename: workdir/0000_xpkg_a/crd.yaml\n\nfrom __future__ import annotations\n\n# filename: in the body stays\n" + files := map[string]string{ + "models/io/k8s/meta/v1.py": withHeader, + "models/io/k8s/meta/__init__.py": "", + "models/README.txt": "# filename: not python\n", + "models/late.py": "a\nb\nc\nd\ne\nf\ng\n# filename: too far down\n", + } + for p, c := range files { + if err := afero.WriteFile(fsys, p, []byte(c), 0o644); err != nil { + t.Fatal(err) + } + } + + if err := stripCodegenFilenameHeaders(fsys, "models"); err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "models/io/k8s/meta/v1.py": "# generated by datamodel-codegen:\n\nfrom __future__ import annotations\n\n# filename: in the body stays\n", + "models/io/k8s/meta/__init__.py": "", + "models/README.txt": files["models/README.txt"], + "models/late.py": files["models/late.py"], + } + for p, w := range want { + got, err := afero.ReadFile(fsys, p) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(w, string(got)); diff != "" { + t.Errorf("%s (-want +got):\n%s", p, diff) + } + } +} diff --git a/internal/schemas/generator/rust.go b/internal/schemas/generator/rust.go index 54c10885..9f5e2c95 100644 --- a/internal/schemas/generator/rust.go +++ b/internal/schemas/generator/rust.go @@ -17,9 +17,12 @@ limitations under the License. package generator import ( + "bytes" "context" + "io/fs" "maps" "path" + "path/filepath" "slices" "strconv" "strings" @@ -102,6 +105,13 @@ func (rustGenerator) Language() string { return devv1alpha1.SchemaLanguageRust } +// Fingerprint reports that this generator has no output-affecting options. It +// runs in-process, so its output changes only with the CLI version, which the +// lock fingerprints separately. +func (rustGenerator) Fingerprint() string { + return "" +} + // GenerateFromCRD generates Rust models for the XRDs and CRDs in the given // filesystem. func (rustGenerator) GenerateFromCRD(_ context.Context, fromFS afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { @@ -897,6 +907,62 @@ func rustWriteDoc(sb *strings.Builder, indent, doc string) { } } +// MergeGeneratedSchemas combines the crates generated from several sources +// into one. Each source's crate declares only its own modules, in src/lib.rs, +// the mod.rs files and Cargo.toml's features, so those files differ between +// sources and are rebuilt for the merged crate instead of copied. Every other +// file is a model, which two sources may both emit, such as the shared +// Kubernetes types, but must emit identically. +func (rustGenerator) MergeGeneratedSchemas(parts []afero.Fs) (afero.Fs, error) { + out := afero.NewMemMapFs() + crateFS := afero.NewBasePathFs(out, rustModelsDir) + for _, part := range parts { + if part == nil { + continue + } + partCrate := afero.NewBasePathFs(part, rustModelsDir) + err := afero.Walk(partCrate, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() || rustModuleTreeFile(p) { + return err + } + content, err := afero.ReadFile(partCrate, p) + if err != nil { + return err + } + existing, err := afero.ReadFile(crateFS, p) + switch { + case err == nil: + if !bytes.Equal(existing, content) { + return errors.Errorf("conflicting generated Rust file %s: two sources produced different content for it", p) + } + return nil + case !errors.Is(err, fs.ErrNotExist): + return err + } + if err := crateFS.MkdirAll(path.Dir(p), 0o755); err != nil { + return err + } + return afero.WriteFile(crateFS, p, content, 0o644) + }) + if err != nil { + return nil, err + } + } + if err := BuildRustModuleTree(crateFS); err != nil { + return nil, err + } + return out, nil +} + +// rustModuleTreeFile reports whether a path in a generated crate is one that +// BuildRustModuleTree writes: the manifest, src/lib.rs, or a mod.rs. No model +// file is named like these, since "mod" and "lib" are escaped (see +// rustFileStem). +func rustModuleTreeFile(p string) bool { + p = filepath.ToSlash(p) + return p == "Cargo.toml" || p == rustSrcDir+"/lib.rs" || path.Base(p) == "mod.rs" +} + // BuildRustModuleTree rewrites everything about the generated Rust crate rooted // at crateFS that depends on which models it holds: the module declarations in // src/lib.rs and in one mod.rs per directory, and the manifest, whose features diff --git a/internal/schemas/generator/rust_merge_test.go b/internal/schemas/generator/rust_merge_test.go new file mode 100644 index 00000000..41ec5c9b --- /dev/null +++ b/internal/schemas/generator/rust_merge_test.go @@ -0,0 +1,102 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/spf13/afero" +) + +func TestRustMergeGeneratedSchemas(t *testing.T) { + t.Parallel() + + part := func(t *testing.T, files map[string]string) afero.Fs { + t.Helper() + fsys := afero.NewMemMapFs() + for p, content := range files { + if err := afero.WriteFile(fsys, p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return fsys + } + + cases := map[string]struct { + reason string + parts []map[string]string + // wantErr is cmpopts.AnyError if merging must fail. + wantErr error + // errMsg is part of the error message. The error is built with + // errors.Errorf, so there is no sentinel to compare against, and the + // message naming the conflicting file is the behavior under test. + errMsg string + // wantLib are lines src/lib.rs must contain. + wantLib []string + }{ + "ModuleTreesAreRebuilt": { + reason: "Each source's lib.rs, mod.rs and Cargo.toml describe only that source, so they must be rebuilt rather than conflict.", + parts: []map[string]string{ + {"models/Cargo.toml": "a", "models/src/lib.rs": "pub mod com;", "models/src/com/mod.rs": "a", "models/src/com/widget.rs": "W"}, + {"models/Cargo.toml": "b", "models/src/lib.rs": "pub mod io;", "models/src/io/mod.rs": "b", "models/src/io/bucket.rs": "B"}, + }, + wantLib: []string{"pub mod com;", "pub mod io;"}, + }, + "SharedModelsMustMatch": { + reason: "A model two sources emit must be byte-identical.", + parts: []map[string]string{ + {"models/src/io/objectmeta.rs": "one"}, + {"models/src/io/objectmeta.rs": "two"}, + }, + wantErr: cmpopts.AnyError, + errMsg: "conflicting generated Rust file src/io/objectmeta.rs", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + parts := make([]afero.Fs, 0, len(tc.parts)) + for _, files := range tc.parts { + parts = append(parts, part(t, files)) + } + merged, err := rustGenerator{}.MergeGeneratedSchemas(parts) + if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nMergeGeneratedSchemas(...): -want error, +got error:\n%s", tc.reason, diff) + } + if err != nil { + if !strings.Contains(err.Error(), tc.errMsg) { + t.Errorf("%s\nMergeGeneratedSchemas(...): error %q does not contain %q", tc.reason, err, tc.errMsg) + } + return + } + lib, err := afero.ReadFile(merged, "models/src/lib.rs") + if err != nil { + t.Fatal(err) + } + for _, want := range tc.wantLib { + if !strings.Contains(string(lib), want) { + t.Errorf("%s\nsrc/lib.rs lacks %q:\n%s", tc.reason, want, lib) + } + } + }) + } +} diff --git a/internal/schemas/manager/lock.go b/internal/schemas/manager/lock.go index e4387d97..90ee2615 100644 --- a/internal/schemas/manager/lock.go +++ b/internal/schemas/manager/lock.go @@ -16,16 +16,170 @@ limitations under the License. package manager -const lockFileName = ".lock.json" +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "maps" + "slices" -// lock tracks the versions of sources whose schemas are present in the -// manager. It is persisted to the manager's filesystem. + "github.com/spf13/afero" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + crversion "github.com/crossplane/crossplane-runtime/v2/pkg/version" +) + +const ( + lockFileName = ".lock.json" + + // lockVersion is the format of the lock file this CLI writes. A lock with + // a different (or missing) version is not understood, and reads as stale + // so the project regenerates rather than trusting it. + lockVersion = 2 + + // sourceGroupDependencies is the group of every source the project's + // dependencies contribute: each declared dependency and the transitive + // xpkg dependencies their metadata declares. + sourceGroupDependencies = "dependencies" + + // sourceGroupAPIs is the group of the project's own apis/ directory. + sourceGroupAPIs = "apis" +) + +// lock tracks the state of the generated schema tree. It is persisted to the +// manager's filesystem, and read back to decide whether the tree is fresh. +// +// The tree is assembled from per-group output caches (see the layout in +// manager.go). A group is fresh only when all of these hold: +// +// - its source map exactly matches the current sources, so a removed +// dependency is stale, +// - the language set matches the configured generators, +// - the generator fingerprint matches, so a CLI or toolchain upgrade that +// changes generated code is stale, +// - its cached output still exists. type lock struct { - Packages map[string]string `json:"packages"` + // Version of the lock format. Anything other than lockVersion is not + // understood and reads as stale. + Version int `json:"version"` + + // Generator is a fingerprint of everything outside the sources that + // determines what the generators produce: the CLI build version and each + // generator's own fingerprint (its pinned toolchain image and options). + Generator string `json:"generator,omitempty"` + + // Languages the schemas on disk were generated for, sorted. + Languages []string `json:"languages,omitempty"` + + // Groups maps each source group name to the source ID to version map the + // group was last generated from. + Groups map[string]map[string]string `json:"groups"` +} + +// groupFresh reports whether a group's cached output was generated from +// exactly these sources. An absent group -- a lock from before groups +// existed -- is never fresh. +func (l *lock) groupFresh(group string, sources map[string]string) bool { + if l == nil { + return false + } + return maps.Equal(l.Groups[group], sources) +} + +// equal reports whether two locks record the same state. +func (l *lock) equal(o *lock) bool { + return l.Version == o.Version && + l.Generator == o.Generator && + slices.Equal(l.Languages, o.Languages) && + maps.EqualFunc(l.Groups, o.Groups, maps.Equal) +} + +// sourceMap builds the source ID to version map a freshness check compares +// against, from each source's recorded version. +func sourceMap(sources []Source, versions map[string]string) map[string]string { + out := make(map[string]string, len(sources)) + for _, src := range sources { + out[src.ID()] = versions[src.ID()] + } + return out +} + +// readLock reads the persisted lock. A lock that cannot be read -- missing, +// unparseable, truncated by an interrupted write, or in a different format +// version -- reads as an empty lock, which means stale. The response to a +// lock we cannot trust is regeneration, not an error: every command that +// writes schemas can produce a fresh tree, so failing the build over the lock +// would only stand in the way of recovering from it. +func (m *Manager) readLock() *lock { + bs, err := afero.ReadFile(m.fs, lockFileName) + if err != nil { + return &lock{} + } + + var l lock + if err := json.Unmarshal(bs, &l); err != nil { + return &lock{} + } + + if l.Version != lockVersion { + return &lock{} + } + + return &l +} + +// writeLock persists the lock atomically -- a temp file first, then a rename +// -- so an interrupted write can never leave a truncated lock: it leaves a +// truncated temp file, which the next run overwrites and the lock itself +// stays the old, stale one. +func (m *Manager) writeLock(l *lock) error { + bs, err := json.MarshalIndent(l, "", " ") + if err != nil { + return errors.Wrap(err, "failed to serialize schema lock") + } + // A trailing newline so the file ends cleanly and edits to the last entry + // produce minimal diffs. + bs = append(bs, '\n') + + if err := m.fs.MkdirAll("/", 0o750); err != nil { + return errors.Wrap(err, "failed to ensure schema directory exists") + } + + tmp := lockFileName + ".tmp" + if err := afero.WriteFile(m.fs, tmp, bs, 0o600); err != nil { + return errors.Wrap(err, "failed to write schema lock file") + } + if err := m.fs.Rename(tmp, lockFileName); err != nil { + return errors.Wrap(err, "failed to put the schema lock file in place; the next build will regenerate the schemas") + } + return nil +} + +// generatorFingerprint derives the lock's generator fingerprint from the +// configured generators. Each generator contributes its language and its own +// Fingerprint() -- its pinned toolchain image and options -- so any change +// that alters what a generator produces also changes the lock's fingerprint +// and invalidates the caches. m.languages() is sorted, so the parts are +// stable for the same generator set. +func (m *Manager) generatorFingerprint() string { + parts := make([]string, 0, len(m.generators)+1) + parts = append(parts, "cli="+crversion.New().GetVersionString()) + for _, g := range m.generators { + parts = append(parts, g.Language()+"="+g.Fingerprint()) + } + return hashParts(parts) } -func newLock() *lock { - return &lock{ - Packages: make(map[string]string), +// hashParts returns a short hex digest of the given strings, each length- +// prefixed so no two different part lists can hash the same. +func hashParts(parts []string) string { + h := sha256.New() + var lenBuf [8]byte + for _, p := range parts { + binary.LittleEndian.PutUint64(lenBuf[:], uint64(len(p))) + _, _ = h.Write(lenBuf[:]) + _, _ = h.Write([]byte(p)) } + return hex.EncodeToString(h.Sum(nil))[:16] } diff --git a/internal/schemas/manager/manager.go b/internal/schemas/manager/manager.go index 72a5331c..9ebe87be 100644 --- a/internal/schemas/manager/manager.go +++ b/internal/schemas/manager/manager.go @@ -18,10 +18,19 @@ limitations under the License. package manager import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "fmt" "io/fs" + "maps" + "path" "path/filepath" + "runtime" + "slices" + "strings" "sync" "github.com/invopop/jsonschema" @@ -36,77 +45,464 @@ import ( "github.com/crossplane/cli/v2/internal/schemas/runner" ) +// The layout under the managed schemas directory: +// +// / the tree users (and function builds) consume +// .cache/// per-group generator output, the assembly inputs +// .cache/tree.json the hash of every file the last assembly wrote +// .staging/ per-run scratch, removed when a pass ends +// +// The dot prefixes keep Go tooling out of the hidden directories, and both sit +// beside the language directories rather than inside them. Each holds a +// .gitignore that ignores everything, so projects that check schemas/ in do +// not pick them up. +const ( + // cacheDirName holds the per-group generator output caches. Each group + // (dependencies, apis) is generated as a unit into here, and the final + // language trees are assembled from these caches, so an XRD edit + // regenerates only the apis group and reuses the dependency output. + cacheDirName = ".cache" + + // stagingDirName holds in-progress generation and assembly. A crash or a + // generation failure leaves only scratch here: the previous tree, the + // caches and the lock stay in place. It is removed when a pass ends, and + // leftovers from a crashed run at the start of the next pass. + stagingDirName = ".staging" + + // treeManifestFileName records, in the cache directory, the SHA-256 of + // every file the last assembly put in each language tree. It lets a pass + // notice a generated file that was deleted or edited by hand, and is kept + // out of the lock so a project that checks schemas/ in does not commit a + // line per generated file. + treeManifestFileName = "tree.json" + + // ignoreAllGitignore is written into the cache and staging directories. + // It ignores everything, itself included. + ignoreAllGitignore = "# Generated by crossplane. Do not check in.\n*\n" +) + // Manager is a schema manager. It manages a directory of schemas, generating // new schemas only when necessary. +// +// A Manager runs one schema-writing pass at a time: the pass reuses the +// manager's staging directory, so two concurrent passes on one Manager would +// corrupt each other's staging. Every command builds a fresh Manager per run, +// which is all the current call sites need. type Manager struct { fs afero.Fs generators []generator.Interface runner runner.SchemaRunner +} - lockMu sync.RWMutex +// GenerateFromMultipleSources generates schemas from every source at once: +// the sources the project's dependencies contribute, and the project's own +// apis/ directory (nil when there is none). The two are generated and cached +// as separate groups, so the caller says which is which rather than the +// manager guessing from source IDs. +// +// Schemas used to be written one source at a time, which could not work for +// a language whose generator describes an entire run in its output +// (TypeScript, #170: one npm package per run), and quietly rotted the tree +// for every language: copying never removed anything, so renamed kinds and +// dropped dependencies left stale models behind, and the lock recorded only +// versions, so changed languages never triggered regeneration. +// +// The pass caches each group's output and assembles the language trees from +// the caches. See generateSources for the full flow and the crash/failure +// semantics. +func (m *Manager) GenerateFromMultipleSources(ctx context.Context, dependencies []Source, apis Source) error { + var apiSources []Source + if apis != nil { + apiSources = []Source{apis} + } + for _, src := range slices.Concat(dependencies, apiSources) { + if src.Type() != SourceTypeCRD && src.Type() != SourceTypeOpenAPI { + return errors.Errorf("cannot generate schemas for %s: source type %q is not supported; use a CRD or OpenAPI source", describeSource(src), src.Type()) + } + } + return m.generateSources(ctx, dependencies, apiSources) } -// Add ensures schemas for resources in the given source are present in the -// managed directory. -func (m *Manager) Add(ctx context.Context, source Source) error { - version, err := source.Version(ctx) - if err != nil { - return err +// generateSources runs one full generation pass over the two source groups. +// +// 1. Remove leftover .staging from a crashed run: it is scratch by +// construction. +// 2. For each group, decide freshness from the stamp in its cache (source +// map, languages, generator fingerprint) and the cache's existence. +// Regenerate the stale groups concurrently into .staging, stamp them, +// then rename into .cache//, so a failure leaves the previous +// cache in place. +// 3. If either group changed, or the on-disk tree does not match what the +// lock describes, assemble each language directory in .staging from the +// group caches and swap it in by rename. +// 4. Write the lock last, as a temp file plus rename. +// 5. Remove .staging, whether the pass succeeded or not. +// +// Until step 3's rename the previous tree stays in place and usable; a +// failure at any point leaves the previous tree, caches and lock untouched. +// A crash between the tree swap and the lock write leaves a new tree with an +// old lock, which the next pass reads as stale and reassembles -- from the +// caches, so that is cheap. +func (m *Manager) generateSources(ctx context.Context, dependencySources, apiSources []Source) error { + if err := m.fs.RemoveAll(stagingDirName); err != nil { + return errors.Wrapf(err, "failed to clean up the %s scratch directory", stagingDirName) } + // Staging is scratch once the pass ends: on success it holds only the + // replaced trees, and a failure has already undone any swap. Removal is + // best effort, because the next pass removes leftovers first anyway. + defer func() { _ = m.fs.RemoveAll(stagingDirName) }() - existing, err := m.currentVersion(source.ID()) - if err != nil { + current := m.readLock() + fingerprint := m.generatorFingerprint() + languages := m.languages() + + type sourceGroup struct { + name string + sources []Source + } + groups := []sourceGroup{ + {name: sourceGroupDependencies, sources: dependencySources}, + {name: sourceGroupAPIs, sources: apiSources}, + } + + // The lock this pass will record: both groups' source maps, filled in as + // each group is processed. versions are computed up front so a failure + // after a group's generation does not leave the lock half-updated. + groupSources := map[string]map[string]string{} + for _, group := range groups { + versions, err := m.sourceVersions(ctx, group.sources) + if err != nil { + return err + } + groupSources[group.name] = sourceMap(group.sources, versions) + } + + // Stale groups generate concurrently: they share nothing but the + // generators, which already run one container per call. A failure in one + // cancels the other; a group that finished first keeps its new cache and + // stamp, which the next pass reuses if its sources still match. + var ( + changed bool + changedMu sync.Mutex + ) + eg, egCtx := errgroup.WithContext(ctx) + for _, group := range groups { + // A group is fresh when its cache's stamp records these sources, + // languages and generator fingerprint, and the cache covers every + // current language. Freshness is read from the cache's own stamp, not + // the lock: a pass that regenerated one group and then failed leaves + // that group's new cache behind with the old lock, and judging the + // cache by the lock would assemble the new output under the old + // sources' entry. + want := groupStamp{Generator: fingerprint, Languages: languages, Sources: groupSources[group.name]} + fresh := m.groupStampMatches(group.name, want) && m.groupCacheExists(group.name) + + if fresh { + continue + } + eg.Go(func() error { + if err := m.generateGroup(egCtx, group.name, group.sources, want); err != nil { + return err + } + changedMu.Lock() + changed = true + changedMu.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { return err } - if existing == version { + + // Even with both groups fresh, the tree may not match the lock: a + // generated file can be deleted or edited by hand, or the pass may have + // crashed after the tree swap but before the lock or manifest write. If + // anything is off, assemble from the (now certainly valid) caches. + if changed || !m.treeMatchesLock(current, groupSources, languages, fingerprint) { + if err := m.assembleTree(); err != nil { + return err + } + } + + next := &lock{ + Version: lockVersion, + Generator: fingerprint, + Languages: languages, + Groups: groupSources, + } + // A pass that changed nothing leaves the lock file alone, so a no-op + // build does not touch the project's files. + if next.equal(current) { return nil } + return m.writeLock(next) +} + +// sourceVersions computes each source's version, keyed by source ID. +func (m *Manager) sourceVersions(ctx context.Context, sources []Source) (map[string]string, error) { + versions := make(map[string]string, len(sources)) + for _, src := range sources { + v, err := src.Version(ctx) + if err != nil { + return nil, errors.Wrapf(err, "cannot load %s", describeSource(src)) + } + versions[src.ID()] = v + } + return versions, nil +} - _, err = m.Generate(ctx, source) - return err +// groupCacheExists reports whether the group's cached output covers every +// current language. A cache that predates a newly added language is missing +// that language's output, so the existence check is per language, not just +// of the directory. +func (m *Manager) groupCacheExists(group string) bool { + cacheGroup := afero.NewBasePathFs(m.fs, path.Join(cacheDirName, group)) + for _, lang := range m.languages() { + ok, err := afero.DirExists(cacheGroup, lang) + if err != nil || !ok { + return false + } + } + return true +} + +// groupStamp records what a group's cache was generated from. It is written +// into the staged cache before the cache is swapped in, so a cache and its +// stamp are always replaced together. +type groupStamp struct { + Generator string `json:"generator"` + Languages []string `json:"languages"` + Sources map[string]string `json:"sources"` } -// Generate generates and returns schemas using the manager's generators, and -// adds them to the manager. Unlike Add, Generate will always generate schemas, -// regardless of whether they're already present in the manager. -func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero.Fs, error) { - version, err := source.Version(ctx) +// groupStampFileName sits beside the language directories in +// .cache//, so it never reaches an assembled tree. +const groupStampFileName = ".stamp.json" + +// groupStampMatches reports whether the group's cache was generated from +// exactly what want describes. A missing or unreadable stamp does not match. +func (m *Manager) groupStampMatches(group string, want groupStamp) bool { + bs, err := afero.ReadFile(m.fs, path.Join(cacheDirName, group, groupStampFileName)) if err != nil { - return nil, err + return false + } + var got groupStamp + if err := json.Unmarshal(bs, &got); err != nil { + return false + } + return got.Generator == want.Generator && + slices.Equal(got.Languages, want.Languages) && + maps.Equal(got.Sources, want.Sources) +} + +// treeMatchesLock reports whether the on-disk language tree is the one the +// lock describes for these groups, languages and fingerprint: the lock +// records them, and every file the last assembly wrote is still there, +// unchanged (see treeIntact). +func (m *Manager) treeMatchesLock(current *lock, groupSources map[string]map[string]string, languages []string, fingerprint string) bool { + if !slices.Equal(current.Languages, languages) || current.Generator != fingerprint { + return false } + for group, sources := range groupSources { + if !current.groupFresh(group, sources) { + return false + } + } + return m.treeIntact(languages) +} + +// treeManifest is the content of the tree manifest: for each language, the +// SHA-256 of every file the last assembly wrote, keyed by its slash-separated +// path in the language tree. +type treeManifest struct { + Files map[string]map[string]string `json:"files"` +} - fromFS, err := source.Resources(ctx) +// treeIntact reports whether every file the last assembly wrote for these +// languages is still on disk with the content it was written with. Files +// that were added to a tree are not checked: tools put things there, such as +// Python's __pycache__ directories, and they don't change the generated +// files. A missing or unreadable manifest -- a fresh clone, a crash before +// the manifest write, a manifest from before it existed -- is not intact, +// which costs one assembly from the caches. +func (m *Manager) treeIntact(languages []string) bool { + bs, err := afero.ReadFile(m.fs, path.Join(cacheDirName, treeManifestFileName)) if err != nil { - return nil, err + return false + } + var tm treeManifest + if err := json.Unmarshal(bs, &tm); err != nil { + return false } + for _, lang := range languages { + files, ok := tm.Files[lang] + if !ok { + return false + } + if ok, err := afero.DirExists(m.fs, lang); err != nil || !ok { + return false + } + for p, want := range files { + content, err := afero.ReadFile(m.fs, path.Join(lang, p)) + if err != nil || hashContent(content) != want { + return false + } + } + } + return true +} - var schemasMu sync.Mutex - schemas := make(map[string]afero.Fs) +// hashTree returns the SHA-256 of every file in fsys, keyed by its +// slash-separated path. +func hashTree(fsys afero.Fs) (map[string]string, error) { + files := map[string]string{} + err := afero.Walk(fsys, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := afero.ReadFile(fsys, p) + if err != nil { + return err + } + files[filepath.ToSlash(p)] = hashContent(content) + return nil + }) + return files, err +} + +// hashContent returns the hex SHA-256 of content. +func hashContent(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} + +// writeTreeManifest persists the tree manifest atomically, like the lock. +func (m *Manager) writeTreeManifest(tm treeManifest) error { + bs, err := json.Marshal(tm) + if err != nil { + return errors.Wrap(err, "failed to serialize the schema tree manifest") + } + if err := m.makeIgnoredDir(cacheDirName); err != nil { + return err + } + p := path.Join(cacheDirName, treeManifestFileName) + if err := afero.WriteFile(m.fs, p+".tmp", bs, 0o644); err != nil { + return errors.Wrap(err, "failed to write the schema tree manifest") + } + if err := m.fs.Rename(p+".tmp", p); err != nil { + return errors.Wrap(err, "failed to put the schema tree manifest in place; the next build will reassemble the schemas") + } + return nil +} + +// generateGroup regenerates one group's output from its sources into the +// group cache. Sources are merged per source type into one generator run, +// and the output is staged then renamed into .cache//, so a failure +// during generation leaves the previous cache in place. +func (m *Manager) generateGroup(ctx context.Context, group string, sources []Source, stamp groupStamp) error { + crdSources, openAPISources, err := splitByType(sources) + if err != nil { + return err + } + + partsByLang, err := m.generateParts(ctx, group, crdSources, openAPISources) + if err != nil { + return err + } + + if err := m.makeIgnoredDir(stagingDirName); err != nil { + return err + } + stageDir := path.Join(stagingDirName, group) + if err := m.fs.MkdirAll(stageDir, 0o750); err != nil { + return errors.Wrapf(err, "failed to create the staging directory for %s", group) + } + + for _, lang := range m.languages() { + langStage := afero.NewBasePathFs(m.fs, path.Join(stageDir, lang)) + if err := langStage.MkdirAll(".", 0o750); err != nil { + return errors.Wrapf(err, "failed to stage %s output for %s", lang, group) + } + parts, err := m.mergeParts(lang, partsByLang[lang]) + if err != nil { + return errors.Wrapf(err, "cannot merge the %s schemas generated from the CRD and OpenAPI sources of %s", lang, groupDescription(group)) + } + if err := copyParts(parts, langStage); err != nil { + return errors.Wrapf(err, "cannot combine the %s schemas generated from the CRD and OpenAPI sources of %s", lang, groupDescription(group)) + } + } + + bs, err := json.Marshal(stamp) + if err != nil { + return errors.Wrapf(err, "failed to serialize the %s cache stamp", group) + } + if err := afero.WriteFile(m.fs, path.Join(stageDir, groupStampFileName), bs, 0o644); err != nil { + return errors.Wrapf(err, "failed to stamp the %s cache", group) + } + + // Swap the staged group output into the cache. The old cache is only + // removed once the new one is complete in staging. + cacheDir := path.Join(cacheDirName, group) + if err := m.makeIgnoredDir(cacheDirName); err != nil { + return err + } + if err := m.fs.RemoveAll(cacheDir); err != nil { + return errors.Wrapf(err, "failed to clear the old %s cache", group) + } + if err := m.fs.Rename(stageDir, cacheDir); err != nil { + return errors.Wrapf(err, "failed to move the generated %s schemas into the cache", group) + } + return nil +} + +// generateParts runs the generators over each of a group's sources and +// returns every language's output parts, CRD sources first, each in source +// order. +func (m *Manager) generateParts(ctx context.Context, group string, crdSources, openAPISources []Source) (map[string][]afero.Fs, error) { + // Each source is its own generator run, and the runs go in parallel. One + // combined run per source type would hand a single toolchain process + // every CRD in the group -- datamodel-codegen is single-threaded, so a + // project with a dozen providers took three times as long as generating + // them side by side. The outputs are collected per language rather than + // written immediately: a language whose generator describes the whole + // run in its output merges them into one coherent tree, and the + // languages that produced no output for this group still need an empty + // directory, so the cache always covers the full language set. + type job struct { + t SourceType + index int // position among the group's sources of this type + src Source + } + jobs := make([]job, 0, len(crdSources)+len(openAPISources)) + for i, src := range crdSources { + jobs = append(jobs, job{t: SourceTypeCRD, index: i, src: src}) + } + for i, src := range openAPISources { + jobs = append(jobs, job{t: SourceTypeOpenAPI, index: i, src: src}) + } + + // Indexed by job, so the parts keep CRD-then-OpenAPI, source order no + // matter which run finishes first. + results := make([]map[string]afero.Fs, len(jobs)) eg, egCtx := errgroup.WithContext(ctx) - sourceType := source.Type() - for _, gen := range m.generators { + eg.SetLimit(max(runtime.NumCPU(), 2)) + for i, j := range jobs { eg.Go(func() error { - var schemaFS afero.Fs - var err error - - switch sourceType { - case SourceTypeCRD: - schemaFS, err = gen.GenerateFromCRD(egCtx, fromFS, m.runner) - case SourceTypeOpenAPI: - schemaFS, err = gen.GenerateFromOpenAPI(egCtx, fromFS, m.runner) - default: - return errors.Errorf("unsupported source type %q", sourceType) - } + in, err := m.sourceResources(egCtx, j.index, j.src) if err != nil { return err } - - if schemaFS != nil { - schemasMu.Lock() - schemas[gen.Language()] = schemaFS - schemasMu.Unlock() + schemas, err := m.runGenerators(egCtx, in, j.t) + if err != nil { + // Name the dependency that failed; the APIs group is one + // source, the project's own. + what := describeSource(j.src) + if group == sourceGroupAPIs { + what = groupDescription(group) + } + return errors.Wrapf(err, "cannot generate schemas for %s", what) } - + results[i] = schemas return nil }) } @@ -114,37 +510,439 @@ func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero return nil, err } - // Copy generated schemas into our schema repository. Generators produce - // output into models/ — we strip that prefix by copying from models/ into - // the language directory. - for lang, genFS := range schemas { - langFS := afero.NewBasePathFs(m.fs, lang) + partsByLang := map[string][]afero.Fs{} + for _, schemas := range results { + for lang, schemaFS := range schemas { + partsByLang[lang] = append(partsByLang[lang], schemaFS) + } + } + return partsByLang, nil +} + +// groupDescription names a source group for messages. +func groupDescription(group string) string { + switch group { + case sourceGroupDependencies: + return "the project's dependencies" + case sourceGroupAPIs: + return "the project's own APIs" + default: + return "the " + group + " sources" + } +} + +// splitByType partitions sources into CRD and OpenAPI slices. +func splitByType(sources []Source) (crd, openAPI []Source, err error) { + for _, src := range sources { + switch src.Type() { + case SourceTypeCRD: + crd = append(crd, src) + case SourceTypeOpenAPI: + openAPI = append(openAPI, src) + default: + return nil, nil, errors.Errorf("cannot generate schemas for %s: source type %q is not supported; use a CRD or OpenAPI source", describeSource(src), src.Type()) + } + } + return crd, openAPI, nil +} + +// assembleTree builds each language's tree from the group caches and swaps +// it in by rename. A path present in both groups (e.g. shared apimachinery +// models) must have the same content in both; if it doesn't, the pass fails +// rather than letting one group's copy silently win, as the last copy did on +// main. Languages that neither group produced output for get an empty +// directory, and languages recorded in the last lock but no longer generated +// have their directories removed. +func (m *Manager) assembleTree() error { + // The languages to remove a tree for: everything on disk's language set + // (the lock's Languages, plus the current set) other than what this + // manager still generates. Dropped languages need no assembly -- their + // cache entries went with the last group regeneration -- only removal. + generated := m.languages() + var removed []string + for _, lang := range m.generatedOrRecordedLanguages() { + if !slices.Contains(generated, lang) { + removed = append(removed, lang) + } + } + + if err := m.makeIgnoredDir(stagingDirName); err != nil { + return err + } + + manifest := treeManifest{Files: map[string]map[string]string{}} + for _, lang := range generated { + files, err := m.stageTree(lang) + if err != nil { + return err + } + manifest.Files[lang] = files + } + + // Swap every language directory in by rename. Rename over an existing + // directory is not possible, so the old directory is moved aside within + // staging first (it is scratch already) and the staged tree renamed in. + // If any swap fails, the ones already made are undone, so a failure here + // leaves the previous tree in place like a failure at any earlier step. + replacedDir := path.Join(stagingDirName, "replaced") + if err := m.fs.MkdirAll(replacedDir, 0o750); err != nil { + return errors.Wrap(err, "failed to prepare the schema swap") + } + var swaps []treeSwap + for _, lang := range generated { + sw := treeSwap{lang: lang, replaced: path.Join(replacedDir, lang)} + moved, err := m.renameIfExists(lang, sw.replaced) + if err != nil { + return m.undoSwaps(swaps, errors.Wrapf(err, "failed to move the previous %s schemas aside", lang)) + } + sw.hadPrevious = moved + if err := m.fs.Rename(path.Join(stagingDirName, "tree", lang), lang); err != nil { + return m.undoSwaps(append(swaps, sw), errors.Wrapf(err, "failed to swap the %s tree in", lang)) + } + sw.swappedIn = true + swaps = append(swaps, sw) + } + + // Dropped languages: remove their trees outright. + for _, lang := range removed { + if err := m.fs.RemoveAll(lang); err != nil { + return errors.Wrapf(err, "failed to remove the %s schemas of a language the project no longer generates", lang) + } + } + + // The staged trees are in place; the replaced old trees are scratch now + // and go when generateSources removes staging. The manifest is written + // last: until it is, the previous manifest no longer matches the tree, + // so a crash here costs the next pass one assembly from the caches. + return m.writeTreeManifest(manifest) +} + +// stageTree assembles one language's tree in staging from the group caches, +// and returns the hash of every file in it for the tree manifest. +func (m *Manager) stageTree(lang string) (map[string]string, error) { + treeStage := afero.NewBasePathFs(m.fs, path.Join(stagingDirName, "tree", lang)) + // Languages with no output from either group still get an empty staged + // tree: the swap renames it into place, and a rename needs the directory + // to exist. copyTreeWithConflicts only creates directories when it writes + // a file, so an empty tree would not be created otherwise. + if err := treeStage.MkdirAll(".", 0o750); err != nil { + return nil, errors.Wrapf(err, "failed to stage the %s tree", lang) + } + + depCache := m.groupCacheFS(sourceGroupDependencies, lang) + apiCache := m.groupCacheFS(sourceGroupAPIs, lang) + + // A run-level language (see schemaMerger) has its groups' parts merged + // into one coherent tree; a per-kind language is assembled by copying the + // groups in turn with conflict detection. + if _, ok := m.generatorFor(lang).(schemaMerger); ok { + parts, err := m.mergeParts(lang, []afero.Fs{depCache, apiCache}) + if err != nil { + return nil, errors.Wrapf(err, "cannot merge the %s schemas of the project's dependencies and its own APIs", lang) + } + // The merged tree is assembled like any other: the cache holds + // verbatim generator output, and the models/ prefix most generators + // use (Rust and TypeScript included) is stripped here, once, so it + // can never be stripped twice. + if err := copyPartsStrippingModels(parts, treeStage); err != nil { + return nil, errors.Wrapf(err, "failed to assemble the %s tree", lang) + } + } else { + // Dependencies first, apis last, matching main's copy order. The + // cache holds generator output verbatim; this copy defines the + // on-disk layout, stripping the models/ prefix most generators put + // their output under. + if err := copyPartsStrippingModels([]afero.Fs{depCache, apiCache}, treeStage); err != nil { + return nil, errors.Wrapf(err, "cannot combine the %s schemas of the project's dependencies and its own APIs; if the project defines an API that a dependency also provides, remove one of the definitions", lang) + } + } + + if err := postProcessForLanguage(lang, treeStage); err != nil { + return nil, err + } + + files, err := hashTree(treeStage) + return files, errors.Wrapf(err, "failed to hash the %s tree", lang) +} + +// makeIgnoredDir creates one of the manager's hidden directories with a +// .gitignore that ignores its contents, so they stay out of version control +// in projects that check schemas/ in. +func (m *Manager) makeIgnoredDir(dir string) error { + if err := m.fs.MkdirAll(dir, 0o750); err != nil { + return errors.Wrapf(err, "failed to create the %s directory", dir) + } + if err := afero.WriteFile(m.fs, path.Join(dir, ".gitignore"), []byte(ignoreAllGitignore), 0o644); err != nil { + return errors.Wrapf(err, "failed to write the %s .gitignore", dir) + } + return nil +} + +// mergeParts combines the parts of a run-level language (see schemaMerger) +// into one part, so its run-level files describe all of them. Empty parts +// (a group or source type that produced nothing) are dropped first, and a +// single remaining part needs no merging. Parts of any other language are +// returned unchanged, for the caller to copy with conflict detection. +func (m *Manager) mergeParts(lang string, parts []afero.Fs) ([]afero.Fs, error) { + merger, ok := m.generatorFor(lang).(schemaMerger) + if !ok { + return parts, nil + } + var nonEmpty []afero.Fs + for _, part := range parts { + if part == nil { + continue + } + empty, err := isEmptyFs(part) + if err != nil { + return nil, err + } + if !empty { + nonEmpty = append(nonEmpty, part) + } + } + if len(nonEmpty) < 2 { + return nonEmpty, nil + } + merged, err := merger.MergeGeneratedSchemas(nonEmpty) + if err != nil { + return nil, err + } + return []afero.Fs{merged}, nil +} + +// errFoundFile stops isEmptyFs's walk at the first file; afero.Walk does not +// support filepath.SkipAll. +var errFoundFile = errors.New("found a file") + +// isEmptyFs reports whether fsys holds no files. +func isEmptyFs(fsys afero.Fs) (bool, error) { + err := afero.Walk(fsys, ".", func(_ string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + return errFoundFile + } + return nil + }) + switch { + case errors.Is(err, errFoundFile): + return false, nil + case err != nil: + return false, err + default: + return true, nil + } +} + +// groupCacheFS returns the cached output filesystem for one group and +// language. The cache only ever holds this manager's languages (each group +// regeneration rewrites it for the current set), so it exists for exactly +// the languages assemblable here. +func (m *Manager) groupCacheFS(group, lang string) afero.Fs { + return afero.NewBasePathFs(m.fs, path.Join(cacheDirName, group, lang)) +} - // Try to copy from models/ subdirectory first (generators put output there). - modelsFS := afero.NewBasePathFs(genFS, "models") - hasModels := false - if fi, err := modelsFS.Stat("."); err == nil && fi.IsDir() { - hasModels = true +// generatedOrRecordedLanguages returns the union of the current language set +// and the last lock's, so a dropped language is noticed and its tree +// removed. +func (m *Manager) generatedOrRecordedLanguages() []string { + langs := m.languages() + for _, lang := range m.readLock().Languages { + if !slices.Contains(langs, lang) { + langs = append(langs, lang) } + } + return langs +} + +// treeSwap records one language directory's swap, so a failed swap can be +// undone. +type treeSwap struct { + lang string + replaced string // where the previous tree was moved aside to + hadPrevious bool // a previous tree existed and was moved aside + swappedIn bool // the new tree was renamed into place +} - if hasModels { - if err := filesystem.CopyFilesBetweenFs(modelsFS, langFS); err != nil { - return nil, err +// undoSwaps reverses swaps, newest first: the new tree is removed and the +// previous one moved back. It is best effort -- a language that cannot be +// restored does not stop the others -- and returns cause, plus the languages +// that could not be restored. +func (m *Manager) undoSwaps(swaps []treeSwap, cause error) error { + var failed []string + for _, sw := range slices.Backward(swaps) { + if sw.swappedIn { + if err := m.fs.RemoveAll(sw.lang); err != nil { + failed = append(failed, fmt.Sprintf("%s (%v)", sw.lang, err)) + continue } - } else { - if err := filesystem.CopyFilesBetweenFs(genFS, langFS); err != nil { - return nil, err + } + if sw.hadPrevious { + if err := m.fs.Rename(sw.replaced, sw.lang); err != nil { + failed = append(failed, fmt.Sprintf("%s (%v)", sw.lang, err)) } } + } + if len(failed) > 0 { + return errors.Errorf("%v; the previous schemas could not be restored for %s, and the next build will regenerate them", cause, strings.Join(failed, ", ")) + } + return cause +} - if err := postProcessForLanguage(lang, langFS); err != nil { - return nil, err +// renameIfExists moves "from" onto "to" when "from" exists, and reports +// whether it did; a missing "from" is not an error (a language that has never +// been generated). +func (m *Manager) renameIfExists(from, to string) (bool, error) { + ok, err := afero.Exists(m.fs, from) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + return true, m.fs.Rename(from, to) +} + +// copyParts copies generated output parts into dst. Parts go in verbatim -- +// the same layout the generator produced, models/ prefix included -- so the +// group cache stores generator output unchanged and exactly one copy step +// (tree assembly) defines the on-disk layout by stripping the models/ prefix. +// +// A run-level part is one coherent tree; the non-nil parts of a per-kind +// language all belong to the same group, and a file conflict between two +// parts is an error rather than a silent last-write-wins. +func copyParts(parts []afero.Fs, dst afero.Fs) error { + for _, part := range parts { + if part == nil { + continue + } + if err := copyTreeWithConflicts(part, dst); err != nil { + return err + } + } + return nil +} + +// copyPartsStrippingModels assembles the on-disk tree from cached generator +// output: most generators put their output under a models/ directory, which +// the tree does not keep (main stripped it during copy, and the tree layout +// is part of the user-visible contract). This is the only place the prefix +// is removed, so it can never be stripped twice. +func copyPartsStrippingModels(parts []afero.Fs, dst afero.Fs) error { + for _, part := range parts { + if part == nil { + continue + } + src := part + if ok, err := afero.DirExists(part, "models"); err == nil && ok { + src = afero.NewBasePathFs(part, "models") } + if err := copyTreeWithConflicts(src, dst); err != nil { + return err + } + } + return nil +} + +// copyTreeWithConflicts copies every file from src into dst, erroring when a +// path already exists in dst with different content. Identical content is +// allowed: some inputs are genuinely shared (KCL's kcl.mod, Go's apimachinery +// models) and are byte-for-byte the same from either group. +func copyTreeWithConflicts(src, dst afero.Fs) error { + return afero.Walk(src, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + content, err := afero.ReadFile(src, p) + if err != nil { + return errors.Wrapf(err, "failed to read %s", p) + } + + existing, err := afero.ReadFile(dst, p) + switch { + case err == nil: + if !bytes.Equal(existing, content) { + return errors.Errorf("conflicting generated file %s: two sources generate it with different content", p) + } + return nil + case errors.Is(err, fs.ErrNotExist): + // Not present in dst yet; write it below. + default: + return errors.Wrapf(err, "failed to read the existing %s", p) + } + + if err := dst.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return errors.Wrapf(err, "failed to create the directory for %s", p) + } + return afero.WriteFile(dst, p, content, 0o644) + }) +} + +// sourceResources returns a source's resources under a prefix naming its +// position and ID, so inputs from different sources can never collide on +// name. +func (m *Manager) sourceResources(ctx context.Context, index int, src Source) (afero.Fs, error) { + srcFS, err := src.Resources(ctx) + if err != nil { + return nil, errors.Wrapf(err, "cannot load %s", describeSource(src)) + } + + in := afero.NewMemMapFs() + prefix := fmt.Sprintf("%04d_%s", index, sanitizeSourceID(src.ID())) + if err := filesystem.CopyFilesBetweenFs(srcFS, afero.NewBasePathFs(in, prefix)); err != nil { + return nil, errors.Wrapf(err, "cannot read the resources of %s", describeSource(src)) + } + return in, nil +} + +// runGenerators runs all generators on the merged filesystem and returns the generated schemas. +func (m *Manager) runGenerators(ctx context.Context, mergedFS afero.Fs, sourceType SourceType) (map[string]afero.Fs, error) { + schemas := make(map[string]afero.Fs) + var schemasMu sync.Mutex + eg, egCtx := errgroup.WithContext(ctx) + + for _, gen := range m.generators { + eg.Go(func() error { + schemaFS, err := m.runGenerator(egCtx, gen, mergedFS, sourceType) + if err != nil { + return errors.Wrapf(err, "%s generator", gen.Language()) + } + if schemaFS != nil { + schemasMu.Lock() + schemas[gen.Language()] = schemaFS + schemasMu.Unlock() + } + return nil + }) } - return schemas, m.updateVersion(source.ID(), version) + if err := eg.Wait(); err != nil { + return nil, err + } + + return schemas, nil +} + +// runGenerator runs a single generator on the merged filesystem. +func (m *Manager) runGenerator(ctx context.Context, gen generator.Interface, mergedFS afero.Fs, sourceType SourceType) (afero.Fs, error) { + switch sourceType { + case SourceTypeCRD: + return gen.GenerateFromCRD(ctx, mergedFS, m.runner) + case SourceTypeOpenAPI: + return gen.GenerateFromOpenAPI(ctx, mergedFS, m.runner) + default: + return nil, errors.Errorf("unsupported source type %q", sourceType) + } } +// postProcessForLanguage runs language-specific finalization over an +// assembled tree (the JSON index schema). func postProcessForLanguage(language string, langFS afero.Fs) error { switch language { case devv1alpha1.SchemaLanguageJSON: @@ -171,6 +969,11 @@ func jsonBuildIndexSchema(langFS afero.Fs) error { } metaFile := "index.schema.json" + if !slices.ContainsFunc(schemas, func(s string) bool { return s != metaFile }) { + // Nothing to index. The tree is assembled fresh, so there is no stale + // index to remove either. + return nil + } var metaSchema jsonschema.Schema for _, schema := range schemas { if schema == metaFile { @@ -188,68 +991,47 @@ func jsonBuildIndexSchema(langFS afero.Fs) error { return afero.WriteFile(langFS, metaFile, bs, 0o644) } -func (m *Manager) currentVersion(id string) (string, error) { - m.lockMu.RLock() - defer m.lockMu.RUnlock() - - l, err := m.getLock() - if err != nil { - return "", err - } - - return l.Packages[id], nil +// schemaMerger is implemented by a generator whose output for one language +// describes an entire generation run rather than one source: the TypeScript +// generator's root index.js/index.d.ts and package.json describe every group +// the run saw, unlike the other generators' per-kind files. When a language +// has output from more than one group in its cache, assembling the tree +// combines them through this interface first, instead of letting the later +// group's copy silently replace the earlier one's root files. +type schemaMerger interface { + MergeGeneratedSchemas(parts []afero.Fs) (afero.Fs, error) } -func (m *Manager) updateVersion(id, version string) error { - m.lockMu.Lock() - defer m.lockMu.Unlock() - - l, err := m.getLock() - if err != nil { - return err +// languages returns the sorted language identifiers this manager generates for. +func (m *Manager) languages() []string { + langs := make([]string, 0, len(m.generators)) + for _, g := range m.generators { + langs = append(langs, g.Language()) } - - l.Packages[id] = version - - return m.updateLock(l) + slices.Sort(langs) + return langs } -func (m *Manager) getLock() (*lock, error) { - lf, err := m.fs.Open(lockFileName) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return newLock(), nil +// generatorFor returns the configured generator for a language, or nil if +// none matches. lang always comes from a generator's own Language(), so a nil +// result would mean the generator set changed mid-pass. +func (m *Manager) generatorFor(lang string) generator.Interface { + for _, g := range m.generators { + if g.Language() == lang { + return g } - return nil, err - } - defer func() { _ = lf.Close() }() - - var l lock - if err := json.NewDecoder(lf).Decode(&l); err != nil { - return nil, err } - - return &l, nil + return nil } -func (m *Manager) updateLock(l *lock) error { - if err := m.fs.MkdirAll("/", 0o750); err != nil { - return errors.Wrap(err, "failed to ensure schema directory exists") - } - - bs, err := json.MarshalIndent(l, "", " ") - if err != nil { - return errors.Wrap(err, "failed to serialize schema lock") +// sanitizeSourceID converts a source ID to a safe directory name. +func sanitizeSourceID(id string) string { + // Replace characters that are problematic in filesystem paths + result := id + for _, c := range []string{"://", ":", "/", "@"} { + result = strings.ReplaceAll(result, c, "_") } - // Append a trailing newline so the file ends cleanly and edits to the last - // entry produce minimal diffs. - bs = append(bs, '\n') - - if err := afero.WriteFile(m.fs, lockFileName, bs, 0o600); err != nil { - return errors.Wrap(err, "failed to write schema lock file") - } - - return nil + return result } // New returns an initialized manager. diff --git a/internal/schemas/manager/manager_rust_test.go b/internal/schemas/manager/manager_rust_test.go index f83afaf5..6990744f 100644 --- a/internal/schemas/manager/manager_rust_test.go +++ b/internal/schemas/manager/manager_rust_test.go @@ -17,7 +17,7 @@ limitations under the License. package manager import ( - "context" + "fmt" "regexp" "testing" @@ -26,108 +26,90 @@ import ( devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/runner" ) var rustModDeclRE = regexp.MustCompile(`(?m)^(?:pub )?mod (\w+);`) -// TestGenerateRustModuleTreeAcrossSources covers the reason the Rust generator -// leaves its module declarations to the manager: every source's models are -// copied into one crate, and a generator run only ever sees its own source. The -// declarations have to describe the merged crate, or the second source to -// generate leaves a crate that doesn't compile. -func TestGenerateRustModuleTreeAcrossSources(t *testing.T) { +// rustTestCRD returns a minimal CRD for the given group and kind. +func rustTestCRD(group, kind, plural string) string { + return fmt.Sprintf(`apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: %[3]s.%[1]s +spec: + group: %[1]s + names: + kind: %[2]s + listKind: %[2]sList + plural: %[3]s + singular: %[3]s + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + size: + type: integer +`, group, kind, plural) +} + +// TestRustModuleTreeAcrossSources covers why the Rust generator is run-level: +// each source's crate declares only its own modules in src/lib.rs, the mod.rs +// files and Cargo.toml, so copying the sources' crates over each other would +// either conflict or leave declarations that describe one source. The merged +// crate has to declare every source's modules. +func TestRustModuleTreeAcrossSources(t *testing.T) { t.Parallel() - cases := map[string]struct { - reason string - args struct { - // sources is the files the generator writes for each source, in - // the order the sources are generated. - sources [][]string - } - want struct { - // modules is the modules each file declares. - modules map[string][]string - } - }{ - "DependencyExtendsTheProjectsModules": { - reason: "The declarations describe the merged crate, not only the source generated last.", - args: struct{ sources [][]string }{sources: [][]string{ - // The project's own XRDs. - { - "models/src/com/example/v1/widget.rs", - "models/src/io/k8s/apimachinery/pkg/apis/meta/v1/objectmeta.rs", - }, - // A dependency that adds a kind to a module the project already - // generated, plus a module of its own. - { - "models/src/com/example/v1/gadget.rs", - "models/src/io/upbound/aws/s3/v1beta2/bucket.rs", - "models/src/io/k8s/apimachinery/pkg/apis/meta/v1/objectmeta.rs", - }, - }}, - want: struct{ modules map[string][]string }{modules: map[string][]string{ - "rust/src/lib.rs": {"com", "io"}, - "rust/src/com/example/v1/mod.rs": {"gadget", "widget"}, - "rust/src/io/mod.rs": {"k8s", "upbound"}, - "rust/src/io/k8s/apimachinery/pkg/apis/meta/v1/mod.rs": {"objectmeta"}, - "rust/src/io/upbound/aws/s3/v1beta2/mod.rs": {"bucket"}, - }}, - }, + gens := generator.Filter(generator.AllLanguages(), []string{devv1alpha1.SchemaLanguageRust}) + if len(gens) != 1 { + t.Fatalf("want the Rust generator, got %d generators", len(gens)) } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - t.Parallel() - - testFS := afero.NewMemMapFs() - for i, files := range tc.args.sources { - m := New(testFS, []generator.Interface{&rustMockGenerator{files: files}}, nil) - if _, err := m.Generate(t.Context(), &mockSource{id: "source", version: "v1.0.0"}); err != nil { - t.Fatalf("\n%s\nGenerate(...) for source %d: %v", tc.reason, i, err) - } - } - - for path, want := range tc.want.modules { - contents, err := afero.ReadFile(testFS, path) - if err != nil { - t.Errorf("\n%s\nreading %s: %v", tc.reason, path, err) - continue - } - decls := rustModDeclRE.FindAllStringSubmatch(string(contents), -1) - got := make([]string, 0, len(decls)) - for _, m := range decls { - got = append(got, m[1]) - } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("\n%s\n%s: -want modules, +got modules:\n%s", tc.reason, path, diff) - } - } - }) + // The project's own API, and two dependencies: one adds a kind to the + // project's module and a module of its own, the other shares that module. + api := &mockSource{id: "fs://apis", version: "v1", resources: map[string]string{ + "widget.yaml": rustTestCRD("example.com", "Widget", "widgets"), + }} + depA := &mockSource{id: "xpkg://example/a", version: "v1", resources: map[string]string{ + "gadget.yaml": rustTestCRD("example.com", "Gadget", "gadgets"), + "bucket.yaml": rustTestCRD("s3.aws.example.io", "Bucket", "buckets"), + }} + depB := &mockSource{id: "xpkg://example/b", version: "v1", resources: map[string]string{ + "queue.yaml": rustTestCRD("s3.aws.example.io", "Queue", "queues"), + }} + + testFS := afero.NewMemMapFs() + m := New(testFS, gens, nil) + if err := m.GenerateFromMultipleSources(t.Context(), []Source{depA, depB}, api); err != nil { + t.Fatalf("GenerateFromMultipleSources(...): %v", err) } -} - -// rustMockGenerator writes the given files, as the Rust generator does for the -// schemas of one source. -type rustMockGenerator struct { - files []string -} - -func (g *rustMockGenerator) Language() string { - return devv1alpha1.SchemaLanguageRust -} -func (g *rustMockGenerator) GenerateFromCRD(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { - fs := afero.NewMemMapFs() - for _, path := range g.files { - if err := afero.WriteFile(fs, path, []byte("// Code generated by github.com/crossplane/cli/v2 DO NOT EDIT.\n"), 0o600); err != nil { - return nil, err + want := map[string][]string{ + "rust/src/lib.rs": {"com", "io"}, + "rust/src/com/example/v1/mod.rs": {"gadget", "gadgetlist", "widget", "widgetlist"}, + "rust/src/io/example/aws/s3/v1/mod.rs": {"bucket", "bucketlist", "queue", "queuelist"}, + } + for path, wantMods := range want { + contents, err := afero.ReadFile(testFS, path) + if err != nil { + t.Errorf("reading %s: %v", path, err) + continue + } + decls := rustModDeclRE.FindAllStringSubmatch(string(contents), -1) + got := make([]string, 0, len(decls)) + for _, d := range decls { + got = append(got, d[1]) + } + if diff := cmp.Diff(wantMods, got); diff != "" { + t.Errorf("%s: -want modules, +got modules:\n%s", path, diff) } } - return fs, nil -} - -func (g *rustMockGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { - return nil, nil } diff --git a/internal/schemas/manager/manager_test.go b/internal/schemas/manager/manager_test.go index 62e1b18c..54f8c499 100644 --- a/internal/schemas/manager/manager_test.go +++ b/internal/schemas/manager/manager_test.go @@ -18,230 +18,1243 @@ package manager import ( "context" + "encoding/json" + "errors" + "fmt" "io/fs" + "path" + "slices" + "strings" + "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/spf13/afero" + "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/schemas/runner" ) -func TestManager_Add(t *testing.T) { - t.Parallel() +// mockSource is a Source over canned resources and versions. +type mockSource struct { + id string + version string + resources map[string]string + sourceType SourceType +} - tcs := map[string]struct { - lock *lock - gen generator.Interface - src Source +func (s *mockSource) ID() string { return s.id } - expectedLock *lock - expectedFiles map[string]string - expectErr bool - }{ - // Version already matches: skip generation. - "AlreadyGenerated": { - lock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, - }, - gen: &mockGenerator{ - files: map[string]string{ - "should-not-exist": "does not get created", - }, - }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.0.0", - }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, - }, - }, - // No lock at all: generate and record version. - "EmptyLock": { - gen: &mockGenerator{ - files: map[string]string{ - "should-exist": "does get created", - }, - }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.0.0", - }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, - }, - expectedFiles: map[string]string{ - "mock/should-exist": "does get created", - }, - }, - // Version changed: regenerate and update lock. - "VersionUpdated": { - lock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, - }, - gen: &mockGenerator{ - files: map[string]string{ - "should-exist": "does get created", - }, - }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.1.0", - }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.1.0", - }, - }, - expectedFiles: map[string]string{ - "mock/should-exist": "does get created", - }, - }, +func (s *mockSource) Version(_ context.Context) (string, error) { return s.version, nil } + +func (s *mockSource) Resources(_ context.Context) (afero.Fs, error) { + fsys := afero.NewMemMapFs() + for path, contents := range s.resources { + if err := afero.WriteFile(fsys, path, []byte(contents), 0o600); err != nil { + return nil, err + } } + return fsys, nil +} - for name, tc := range tcs { - t.Run(name, func(t *testing.T) { - t.Parallel() +func (s *mockSource) Type() SourceType { + if s.sourceType == "" { + return SourceTypeCRD + } + return s.sourceType +} - testFS := afero.NewMemMapFs() +// walkInputFiles returns the sorted file paths (not directories) under in. +func walkInputFiles(in afero.Fs) ([]string, error) { + var names []string + err := afero.Walk(in, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + names = append(names, p) + } + return nil + }) + return names, err +} - m := New(testFS, []generator.Interface{tc.gen}, nil) - if tc.lock != nil { - if err := m.updateLock(tc.lock); err != nil { - t.Fatal(err) - } - } +// kindGenerator models a per-kind generator (the shape of the Go, JSON, KCL +// and Python generators): one output file per input resource, so two +// groups' outputs are disjoint -- plus a constant shared root file, which +// some real generators also emit (KCL's kcl.mod) and which assembly must +// accept when its content is identical from both groups. +type kindGenerator struct { + mu sync.Mutex // the manager runs groups and source types concurrently + lang string + fail bool + finger string + runs int // every generation run, both groups + inputs []string // every input file this generator has ever seen + shared string // content of the shared root file; constant across groups + noshare bool // suppress the shared root file + shareFn func(in afero.Fs) (string, error) +} - err := m.Add(t.Context(), tc.src) - if tc.expectErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } +// emitSharedDefault is the default shared root file content, standing in for +// files some real generators write from either group identically (KCL's +// kcl.mod, Go's apimachinery models). +const emitSharedDefault = "shared from either group" + +func (g *kindGenerator) Language() string { + if g.lang == "" { + return "mock" + } + return g.lang +} + +func (g *kindGenerator) Fingerprint() string { return g.finger } + +func (g *kindGenerator) GenerateFromCRD(_ context.Context, in afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + g.mu.Lock() + fail := g.fail + if !fail { + g.runs++ + } + g.mu.Unlock() + if fail { + return nil, errors.New("mock generation failed") + } + + names, err := walkInputFiles(in) + if err != nil { + return nil, err + } + g.mu.Lock() + g.inputs = append(g.inputs, names...) + g.mu.Unlock() + + out := afero.NewMemMapFs() + for _, name := range names { + bs, err := afero.ReadFile(in, name) + if err != nil { + return nil, err + } + if err := afero.WriteFile(out, name, bs, 0o600); err != nil { + return nil, err + } + } + + if !g.noshare { + shared := g.shared + if shared == "" { + shared = emitSharedDefault + } + if g.shareFn != nil { + shared, err = g.shareFn(in) if err != nil { - t.Fatal(err) + return nil, err } + } + if shared != "" { + if err := afero.WriteFile(out, "shared-root", []byte(shared), 0o600); err != nil { + return nil, err + } + } + } + return out, nil +} - _ = afero.Walk(testFS, ".", func(path string, info fs.FileInfo, err error) error { - if err != nil { - t.Fatal(err) - } - if info.Name() == lockFileName { - return nil - } - if info.IsDir() { - return nil - } +func (g *kindGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + return nil, nil +} - want, ok := tc.expectedFiles[path] - if !ok { - t.Errorf("unexpected file %q generated", path) - } +// runLevelGenerator models a generator whose output describes its entire +// input (the TypeScript shape: a root index naming everything the run saw) +// and which implements schemaMerger to combine groups. +type runLevelGenerator struct { + kindGenerator +} - got, err := afero.ReadFile(testFS, path) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(want, string(got)); diff != "" { - t.Errorf("file %q content (-want +got):\n%s", path, diff) - } +func (g *runLevelGenerator) GenerateFromOpenAPI(ctx context.Context, in afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + return g.GenerateFromCRD(ctx, in, nil) +} - return nil - }) +func (g *runLevelGenerator) GenerateFromCRD(_ context.Context, in afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + g.mu.Lock() + fail := g.fail + if !fail { + g.runs++ + } + g.mu.Unlock() + if fail { + return nil, errors.New("mock generation failed") + } - gotLock, err := m.getLock() - if err != nil { + names, err := walkInputFiles(in) + if err != nil { + return nil, err + } + slices.Sort(names) + + out := afero.NewMemMapFs() + if err := afero.WriteFile(out, "index", []byte(strings.Join(names, ",")), 0o600); err != nil { + return nil, err + } + return out, nil +} + +func (g *runLevelGenerator) MergeGeneratedSchemas(parts []afero.Fs) (afero.Fs, error) { + var names []string + for _, part := range parts { + bs, err := afero.ReadFile(part, "index") + if err != nil { + return nil, err + } + if s := string(bs); s != "" { + names = append(names, strings.Split(s, ",")...) + } + } + slices.Sort(names) + + out := afero.NewMemMapFs() + if err := afero.WriteFile(out, "index", []byte(strings.Join(names, ",")), 0o600); err != nil { + return nil, err + } + return out, nil +} + +func readFile(t *testing.T, fsys afero.Fs, p string) string { + t.Helper() + bs, err := afero.ReadFile(fsys, p) + if err != nil { + t.Fatalf("read %s: %v", p, err) + } + return string(bs) +} + +// assertFilesPresent fails the test if any of the given paths does not exist +// under fsys. +func assertFilesPresent(t *testing.T, fsys afero.Fs, paths ...string) { + t.Helper() + for _, p := range paths { + ok, err := afero.Exists(fsys, p) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Errorf("%s is missing", p) + } + } +} + +// assertFilesAbsent fails the test if any of the given paths exists. +func assertFilesAbsent(t *testing.T, fsys afero.Fs, paths ...string) { + t.Helper() + for _, p := range paths { + ok, err := afero.Exists(fsys, p) + if err != nil { + t.Fatal(err) + } + if ok { + t.Errorf("%s should have been removed", p) + } + } +} + +// depSrc builds a dependency-source shorthand. +func depSrc(name, version, file string) *mockSource { + return &mockSource{id: "xpkg://" + name, version: version, resources: map[string]string{file: file}} +} + +// treeDepPath is where a dependency source's file lands in the assembled +// tree: sources are merged under a positional prefix (see +// collectSourceResources), and per-kind generators preserve input paths in +// their output. +func treeDepPath(i int, name string, file string) string { + return fmt.Sprintf("mock/%04d_xpkg_%s/%s", i, strings.ReplaceAll(strings.ReplaceAll(name, ":", "_"), "/", "_"), file) +} + +// treeAPIPath is where the apis source's file lands in the assembled tree. +func treeAPIPath(i int, file string) string { + return fmt.Sprintf("mock/%04d_fs_apis/%s", i, file) +} + +// apiSrc builds the project's apis source shorthand. +func apiSrc(version, file string) *mockSource { + return &mockSource{id: "fs://apis", version: version, resources: map[string]string{file: file}} +} + +// One full pass assembles the tree from both groups, and a second pass over +// unchanged sources generates nothing. +func TestGenerateFromMultipleSources(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml"), treeAPIPath(0, "xrd.yaml"), "mock/shared-root") + + // Both groups ran once. + if got, want := g.runs, 2; got != want { + t.Errorf("generation ran %d times, want %d (once per group)", got, want) + } + + // A second pass over unchanged sources regenerates nothing. + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + if got, want := g.runs, 2; got != want { + t.Errorf("generation ran %d times after an unchanged pass, want still %d", g.runs, want) + } +} + +// An edit to the project's own XRDs must not re-run the dependency group's +// generation: the dependencies cache holds, and only the apis group +// regenerates. On main this regenerates every dependency on every XRD edit. +func TestXRDEditRegeneratesOnlyAPIsGroup(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + // The XRD changed; the dependency did not. + api2 := apiSrc("v2", "xrd.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api2); err != nil { + t.Fatal(err) + } + + // Only the apis group ran: 2 runs before, 1 more now. + if got, want := g.runs, 3; got != want { + t.Errorf("generation ran %d times after an XRD edit, want %d (the dependency group must reuse its cache)", g.runs, want) + } + + // The assembled tree still carries both groups' files, the dependency's + // straight from its cache. + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml"), treeAPIPath(0, "xrd.yaml")) +} + +// A dependency change regenerates the dependency group and reuses the apis +// cache. +func TestDependencyChangeRegeneratesOnlyDependenciesGroup(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + dep2 := depSrc("prov-a:v2", "v2", "dep-v2.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep2}, api); err != nil { + t.Fatal(err) + } + + // Only the dependency group ran. + if got, want := g.runs, 3; got != want { + t.Errorf("generation ran %d times after a dependency change, want %d (the apis group must reuse its cache)", g.runs, want) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v2", "dep-v2.yaml"), treeAPIPath(0, "xrd.yaml")) + assertFilesAbsent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml")) +} + +// A project with no sources must still assemble an empty tree for each +// language: the swap renames the staged tree into place, and a rename needs +// the directory to exist even when no generator produced any file for it. +func TestNoSourcesStillBuildsEmptyTrees(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock", noshare: true}}, nil) + + if err := m.GenerateFromMultipleSources(ctx, nil, nil); err != nil { + t.Fatalf("generation with no sources failed: %v", err) + } + assertFilesAbsent(t, testFS, "mock/shared-root") +} + +// Removing a dependency from the source list must remove its models from the +// tree: the assembled tree describes exactly the current sources. +func TestRemovedDependencyLeavesNoStaleModels(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + + a := depSrc("prov-a:v1", "v1", "a.yaml") + b := depSrc("prov-b:v1", "v1", "b.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{a, b}, nil); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "a.yaml"), treeDepPath(1, "prov-b:v1", "b.yaml")) + + // prov-b removed from the project. + if err := m.GenerateFromMultipleSources(ctx, []Source{a}, nil); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "a.yaml")) + assertFilesAbsent(t, testFS, treeDepPath(1, "prov-b:v1", "b.yaml")) +} + +// A changed language set must regenerate: the caches and the tree were +// produced for a different generator set. A dropped language must not leave +// its tree behind. +func TestLanguageChangeRegeneratesAndClearsDroppedLanguage(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + two := New(testFS, []generator.Interface{&kindGenerator{}, &kindGenerator{lang: "other"}}, nil) + if err := two.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, "mock/shared-root", "other/shared-root") + + // The same project with "other" dropped from the language set. + one := New(testFS, []generator.Interface{&kindGenerator{}}, nil) + if err := one.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, "mock/shared-root") + assertFilesAbsent(t, testFS, "other/shared-root") + if ok, _ := afero.DirExists(testFS, "other"); ok { + t.Error("schemas for the removed language are still on disk") + } +} + +// A changed generator fingerprint must regenerate, even with identical +// sources and languages: the CLI or a toolchain image changed, so the cached +// output may not be what this CLI would produce. +func TestGeneratorFingerprintChangeRegenerates(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + + g := &kindGenerator{lang: "mock", finger: "img:v1"} + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + // The toolchain was upgraded. + g.finger = "img:v2" + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + if got, want := g.runs, 4; got != want { + t.Errorf("generation ran %d times after a fingerprint change, want %d (both groups regenerate)", g.runs, want) + } +} + +// A hand-deleted language directory must not read as fresh: the lock can +// outlive its output, and the next pass must reassemble from the caches. +func TestDeletedLanguageDirIsReassembled(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + if err := testFS.RemoveAll("mock"); err != nil { + t.Fatal(err) + } + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml"), treeAPIPath(0, "xrd.yaml")) +} + +// A damaged or old-format lock must read as stale and cause regeneration, +// not fail the build. +func TestDamagedLockRegenerates(t *testing.T) { + tcs := map[string]struct{ lockContent string }{ + "Truncated": {lockContent: `{"version": 2, "groups": {"de`}, + "OldFormat": {lockContent: `{"packages": {"xpkg://a": "v1"}}`}, + "EmptyNotJSON": {lockContent: ""}, + } + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + testFS := afero.NewMemMapFs() + if err := afero.WriteFile(testFS, lockFileName, []byte(tc.lockContent), 0o600); err != nil { t.Fatal(err) } - if diff := cmp.Diff(tc.expectedLock, gotLock); diff != "" { - t.Errorf("lock (-want +got):\n%s", diff) + + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + if err := m.GenerateFromMultipleSources(t.Context(), []Source{depSrc("prov-a:v1", "v1", "dep.yaml")}, nil); err != nil { + t.Fatalf("pass with a damaged lock failed: %v", err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml")) + + // And the lock was rewritten into the current format. + l := m.readLock() + if l.Version != lockVersion { + t.Errorf("lock version = %d, want %d", l.Version, lockVersion) } }) } } -func TestUpdateLockIsIndented(t *testing.T) { - // The lock file must be written with one entry per line (indented) so it is - // human-readable and avoids spurious merge conflicts. See issue #168. - fs := afero.NewMemMapFs() - m := New(fs, nil, nil) +// A generation failure must leave the previous tree and lock in place, and +// recovery on the next pass must work. +func TestGenerationFailureKeepsPreviousTree(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + // The apis source changes and the generator starts failing. + g.fail = true + api2 := apiSrc("v2", "xrd.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api2); err == nil { + t.Fatal("expected the failing pass to error") + } + + // The previous tree survived untouched. + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml"), treeAPIPath(0, "xrd.yaml")) + + // So did the lock: the failed pass must not record anything. + l := m.readLock() + if got, want := l.Groups[sourceGroupAPIs]["fs://apis"], "v1"; got != want { + t.Errorf("lock apis version = %q, want %q", got, want) + } + + // Recovery: the generator stops failing; the pass regenerates and + // assembles the new tree. + g.fail = false + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api2); err != nil { + t.Fatal(err) + } + l = m.readLock() + if got, want := l.Groups[sourceGroupAPIs]["fs://apis"], "v2"; got != want { + t.Errorf("lock apis version after recovery = %q, want %q", got, want) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a:v1", "dep.yaml"), treeAPIPath(0, "xrd.yaml")) +} + +// A same-path, different-content collision between the two groups must be an +// error, not a silent last-write-wins. +func TestConflictingGeneratedFileErrors(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + + // The shared root file's content depends on which group's input the + // generator saw, so the two groups' copies differ. + g := &kindGenerator{lang: "mock"} + g.shareFn = func(in afero.Fs) (string, error) { + names, err := walkInputFiles(in) + if err != nil { + return "", err + } + for _, n := range names { + if strings.Contains(n, "dep.yaml") { + return "from-dependencies", nil + } + } + return "from-apis", nil + } + + m := New(testFS, []generator.Interface{g}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + err := m.GenerateFromMultipleSources(t.Context(), []Source{dep}, api) + if err == nil { + t.Fatal("expected a conflict error when both groups emit a different root file") + } + for _, want := range []string{"conflicting generated file", "remove one of the definitions"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error does not say %q: %v", want, err) + } + } +} + +// A run-level generator (schemaMerger, TypeScript's shape) must see both +// groups' output merged, not whichever group was copied last. +func TestRunLevelLanguageMergesGroups(t *testing.T) { + t.Parallel() - l := newLock() - l.Packages["xpkg://pkg.example/bar:v1.0.0"] = "sha256:bbb" - l.Packages["xpkg://pkg.example/foo:v0.5.2"] = "sha256:aaa" + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &runLevelGenerator{kindGenerator{lang: "mock"}} + m := New(testFS, []generator.Interface{g}, nil) - if err := m.updateLock(l); err != nil { - t.Fatalf("updateLock: %v", err) + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + if got, want := readFile(t, testFS, "mock/index"), "0000_fs_apis/xrd.yaml,0000_xpkg_prov-a_v1/dep.yaml"; got != want { + t.Errorf("merged index = %q, want %q (both groups' resources, merged)", got, want) } - got, err := afero.ReadFile(fs, lockFileName) + // And an unchanged second pass does not regenerate. + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + if got, want := g.runs, 2; got != want { + t.Errorf("generation ran %d times, want %d", g.runs, want) + } +} + +// The lock file is written indented, one entry per line, so it is +// human-readable and diff-friendly. See issue #168. +func TestLockIsIndented(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + + if err := m.GenerateFromMultipleSources(t.Context(), []Source{depSrc("pkg.example/foo:v0.5.2", "sha256:aaa", "a.yaml")}, nil); err != nil { + t.Fatal(err) + } + + got, err := afero.ReadFile(testFS, lockFileName) if err != nil { - t.Fatalf("read lock: %v", err) + t.Fatal(err) } - // Map keys are marshalled in sorted order, so this golden is deterministic. + // Map keys are marshalled in sorted order, so this golden is + // deterministic. want := `{ - "packages": { - "xpkg://pkg.example/bar:v1.0.0": "sha256:bbb", - "xpkg://pkg.example/foo:v0.5.2": "sha256:aaa" + "version": 2, + "generator": "` + m.generatorFingerprint() + `", + "languages": [ + "mock" + ], + "groups": { + "apis": {}, + "dependencies": { + "xpkg://pkg.example/foo:v0.5.2": "sha256:aaa" + } } } ` if diff := cmp.Diff(want, string(got)); diff != "" { - t.Errorf("lock file is not indented as expected (-want +got):\n%s", diff) + t.Errorf("lock file (-want +got):\n%s", diff) + } +} + +// A pass that regenerates the dependency group and then fails on the apis +// group leaves the new dependency cache behind with the old lock. A later +// pass must not trust that cache as the old sources' output: it is judged by +// its own stamp, so going back to the old dependency regenerates it. +func TestFailedPassDoesNotPoisonGroupCache(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + g.shareFn = func(in afero.Fs) (string, error) { + names, err := walkInputFiles(in) + if err != nil { + return "", err + } + for _, n := range names { + if strings.Contains(n, "broken.yaml") { + return "", errors.New("mock generation failed on a broken XRD") + } + } + return emitSharedDefault, nil + } + m := New(testFS, []generator.Interface{g}, nil) + + depV1 := depSrc("prov-a", "v1", "dep-v1.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{depV1}, apiSrc("v1", "xrd.yaml")); err != nil { + t.Fatal(err) + } + + // The dependency moves to v2 and the XRD is broken: the dependency group + // regenerates into its cache, then the apis group fails. + depV2 := depSrc("prov-a", "v2", "dep-v2.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{depV2}, apiSrc("v2", "broken.yaml")); err == nil { + t.Fatal("expected the pass with a broken XRD to fail") + } + + // The dependency is reverted to v1 and the XRD fixed. The tree must hold + // v1's models, not the v2 output the failed pass cached. + if err := m.GenerateFromMultipleSources(ctx, []Source{depV1}, apiSrc("v3", "xrd-fixed.yaml")); err != nil { + t.Fatal(err) + } + assertFilesPresent(t, testFS, treeDepPath(0, "prov-a", "dep-v1.yaml"), treeAPIPath(0, "xrd-fixed.yaml")) + assertFilesAbsent(t, testFS, treeDepPath(0, "prov-a", "dep-v2.yaml")) +} + +// openAPIRunLevelSrc is a dependency source of OpenAPI type. +func openAPIRunLevelSrc(name, version, file string) *mockSource { + s := depSrc(name, version, file) + s.sourceType = SourceTypeOpenAPI + return s +} + +// A run-level generator must also merge within a group: a project with both +// CRD and OpenAPI dependencies (an xpkg plus k8s) gives the dependency group +// two parts, whose run-level files describe different inputs. +func TestRunLevelLanguageMergesWithinGroup(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + g := &runLevelGenerator{kindGenerator{lang: "mock"}} + m := New(testFS, []generator.Interface{g}, nil) + + crd := depSrc("prov-a:v1", "v1", "dep.yaml") + openAPI := openAPIRunLevelSrc("k8s:v1.35.0", "v1.35.0", "k8s.json") + + if err := m.GenerateFromMultipleSources(ctx, []Source{crd, openAPI}, nil); err != nil { + t.Fatal(err) + } + if got, want := readFile(t, testFS, "mock/index"), "0000_xpkg_k8s_v1.35.0/k8s.json,0000_xpkg_prov-a_v1/dep.yaml"; got != want { + t.Errorf("merged index = %q, want %q (both source types' resources)", got, want) + } +} + +// A per-kind generator whose CRD and OpenAPI output disagree on a shared file +// fails with an error that names the source types inside the group, not the +// two groups. +func TestConflictWithinGroupNamesSourceTypes(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + g := &conflictingTypesGenerator{kindGenerator{lang: "mock"}} + m := New(testFS, []generator.Interface{g}, nil) + + crd := depSrc("prov-a:v1", "v1", "dep.yaml") + openAPI := openAPIRunLevelSrc("k8s:v1.35.0", "v1.35.0", "k8s.json") + + err := m.GenerateFromMultipleSources(t.Context(), []Source{crd, openAPI}, nil) + if err == nil { + t.Fatal("expected a conflict between the CRD and OpenAPI output") + } + for _, want := range []string{"conflicting generated file shared-root", "CRD and OpenAPI", "dependencies"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err, want) + } + } + if strings.Contains(err.Error(), "dependency and API groups") { + t.Errorf("error blames the groups for a conflict inside one group: %v", err) } } -type mockGenerator struct { - files map[string]string +// conflictingTypesGenerator writes a shared root file whose content depends +// on the source type, like KCL's kcl.mod did before its edition was pinned. +type conflictingTypesGenerator struct { + kindGenerator } -func (g *mockGenerator) Language() string { - return "mock" +func (g *conflictingTypesGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + out := afero.NewMemMapFs() + if err := afero.WriteFile(out, "shared-root", []byte("from openapi"), 0o600); err != nil { + return nil, err + } + return out, nil +} + +// modelsGenerator mimics the Python generator's layout: its output is +// models/{pyproject.toml, models//...}. +type modelsGenerator struct { + kindGenerator } -func (g *mockGenerator) GenerateFromCRD(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { - fs := afero.NewMemMapFs() - for path, contents := range g.files { - if err := afero.WriteFile(fs, path, []byte(contents), 0o600); err != nil { +func (g *modelsGenerator) GenerateFromCRD(_ context.Context, in afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + names, err := walkInputFiles(in) + if err != nil { + return nil, err + } + out := afero.NewMemMapFs() + if err := afero.WriteFile(out, "models/pyproject.toml", []byte("[project]"), 0o600); err != nil { + return nil, err + } + for _, n := range names { + if err := afero.WriteFile(out, "models/models/"+n+".py", []byte(n), 0o600); err != nil { return nil, err } } - return fs, nil + return out, nil } -func (g *mockGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { - return nil, nil +// The generator's outer models/ directory is removed exactly once, at +// assembly. The cache keeps the generator output verbatim, so the inner +// models/ package directory survives; removing the prefix twice flattened +// python/models/... to python/.... +func TestModelsPrefixStrippedOnce(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&modelsGenerator{kindGenerator{lang: "python"}}}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + if err := m.GenerateFromMultipleSources(t.Context(), []Source{dep}, api); err != nil { + t.Fatal(err) + } + + assertFilesPresent(t, testFS, + "python/pyproject.toml", + "python/models/0000_xpkg_prov-a_v1/dep.yaml.py", + "python/models/0000_fs_apis/xrd.yaml.py", + ".cache/dependencies/python/models/pyproject.toml", + ) + assertFilesAbsent(t, testFS, "python/0000_xpkg_prov-a_v1/dep.yaml.py") } -type mockSource struct { - id string - version string +// Errors name a source the way the user wrote it, not by its lock ID: an HTTP +// source's ID is http://, which doubles the scheme. +func TestSourceErrorsUseReadableNames(t *testing.T) { + t.Parallel() + + m := New(afero.NewMemMapFs(), []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + src := NewHTTPSource(v1alpha1.Dependency{ + Type: v1alpha1.DependencyTypeCRD, + HTTP: &v1alpha1.HTTPDependency{URL: "http://127.0.0.1:1/crds.yaml"}, + }) + + err := m.GenerateFromMultipleSources(t.Context(), []Source{src}, nil) + if err == nil { + t.Fatal("expected an error for an unreachable HTTP source") + } + if !strings.Contains(err.Error(), "cannot load http://127.0.0.1:1/crds.yaml") { + t.Errorf("error does not name the URL: %v", err) + } + if strings.Contains(err.Error(), "http://http://") { + t.Errorf("error uses the source's lock ID: %v", err) + } } -func (s *mockSource) ID() string { - return s.id +// A generator failure names the language and the group in plain words. +func TestGeneratorErrorNamesLanguageAndGroup(t *testing.T) { + t.Parallel() + + m := New(afero.NewMemMapFs(), []generator.Interface{&kindGenerator{lang: "python", fail: true}}, nil) + err := m.GenerateFromMultipleSources(t.Context(), nil, apiSrc("v1", "xrd.yaml")) + if err == nil { + t.Fatal("expected the failing generator to fail the pass") + } + if want := "cannot generate schemas for the project's own APIs: python generator: mock generation failed"; err.Error() != want { + t.Errorf("error = %q, want %q", err, want) + } } -func (s *mockSource) Version(_ context.Context) (string, error) { - return s.version, nil +func TestDescribeSource(t *testing.T) { + t.Parallel() + + tcs := map[string]struct { + src Source + want string + }{ + "APIsDir": {src: NewFSSource("apis", afero.NewMemMapFs()), want: "apis/"}, + "APIsRoot": {src: NewFSSource(".", afero.NewMemMapFs()), want: "the project root"}, + "Xpkg": {src: NewXpkgSource("xpkg.example/prov-a:v1.0.0", "sha256:abc", afero.NewMemMapFs()), want: "xpkg.example/prov-a:v1.0.0"}, + "HTTP": {src: NewHTTPSource(v1alpha1.Dependency{ + HTTP: &v1alpha1.HTTPDependency{URL: "https://example.com/crds.yaml"}, + }), want: "https://example.com/crds.yaml"}, + "K8s": {src: NewK8sSource(v1alpha1.Dependency{ + K8s: &v1alpha1.K8sDependency{Version: "v1.35.0"}, + }), want: "Kubernetes API v1.35.0"}, + "NoDescriber": {src: depSrc("prov-a", "v1", "a.yaml"), want: "xpkg://prov-a"}, + } + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + if got := describeSource(tc.src); got != tc.want { + t.Errorf("describeSource() = %q, want %q", got, tc.want) + } + }) + } } -func (s *mockSource) Resources(_ context.Context) (afero.Fs, error) { - return nil, nil +// failRenameFs fails the rename that swaps the staged tree of one language +// into place, once armed. +type failRenameFs struct { + afero.Fs + mu sync.Mutex + failLang string } -func (s *mockSource) Type() SourceType { - return SourceTypeCRD +func (f *failRenameFs) arm(lang string) { + f.mu.Lock() + defer f.mu.Unlock() + f.failLang = lang +} + +func (f *failRenameFs) Rename(oldname, newname string) error { + f.mu.Lock() + lang := f.failLang + f.mu.Unlock() + if lang != "" && strings.TrimPrefix(oldname, "/") == path.Join(stagingDirName, "tree", lang) { + return errors.New("mock rename failure") + } + return f.Fs.Rename(oldname, newname) +} + +// A failure while swapping in the second language must put the first +// language's previous tree back: the tree on disk is never half old, half +// new. +func TestFailedSwapRestoresPreviousTree(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := &failRenameFs{Fs: afero.NewMemMapFs()} + m := New(testFS, []generator.Interface{&kindGenerator{lang: "aaa"}, &kindGenerator{lang: "bbb"}}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, apiSrc("v1", "xrd-v1.yaml")); err != nil { + t.Fatal(err) + } + + // The XRD changes, and swapping in bbb's new tree fails after aaa's has + // already been swapped. + testFS.arm("bbb") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, apiSrc("v2", "xrd-v2.yaml")); err == nil { + t.Fatal("expected the failed swap to fail the pass") + } + for _, lang := range []string{"aaa", "bbb"} { + assertFilesPresent(t, testFS, lang+"/0000_fs_apis/xrd-v1.yaml") + assertFilesAbsent(t, testFS, lang+"/0000_fs_apis/xrd-v2.yaml") + } + if ok, _ := afero.Exists(testFS, stagingDirName); ok { + t.Errorf("%s left behind after a failed pass", stagingDirName) + } + + // Once renames work again, the next pass completes the update. + testFS.arm("") + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, apiSrc("v2", "xrd-v2.yaml")); err != nil { + t.Fatal(err) + } + for _, lang := range []string{"aaa", "bbb"} { + assertFilesPresent(t, testFS, lang+"/0000_fs_apis/xrd-v2.yaml") + assertFilesAbsent(t, testFS, lang+"/0000_fs_apis/xrd-v1.yaml") + } +} + +// Sources are grouped by the argument they are passed in, not by their IDs: +// a dependency whose ID happens to look like a local path is still a +// dependency. +func TestGroupsComeFromArguments(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + + vendored := &mockSource{id: "fs://vendored-crds", version: "v1", resources: map[string]string{"crd.yaml": "crd"}} + if err := m.GenerateFromMultipleSources(t.Context(), []Source{vendored}, apiSrc("v1", "xrd.yaml")); err != nil { + t.Fatal(err) + } + + l := m.readLock() + if _, ok := l.Groups[sourceGroupDependencies]["fs://vendored-crds"]; !ok { + t.Errorf("dependency group = %v, want it to hold fs://vendored-crds", l.Groups[sourceGroupDependencies]) + } + if diff := cmp.Diff(map[string]string{"fs://apis": "v1"}, l.Groups[sourceGroupAPIs]); diff != "" { + t.Errorf("apis group (-want +got):\n%s", diff) + } +} + +// A pass that changes nothing does not rewrite the lock file. The lock is +// re-saved in a different (compact) format between the passes; an untouched +// file keeps that format. +func TestUnchangedPassLeavesLockFileAlone(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + compact, err := json.Marshal(m.readLock()) + if err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(testFS, lockFileName, compact, 0o600); err != nil { + t.Fatal(err) + } + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + if got := readFile(t, testFS, lockFileName); got != string(compact) { + t.Errorf("an unchanged pass rewrote the lock file:\n%s", got) + } + + // A change still writes it. + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, apiSrc("v2", "xrd.yaml")); err != nil { + t.Fatal(err) + } + if got := readFile(t, testFS, lockFileName); got == string(compact) { + t.Error("a changed pass did not rewrite the lock file") + } +} + +// Each source is its own generator run, so a group of many dependencies +// generates in parallel instead of in one long toolchain process; the parts +// still assemble into one tree. +func TestEachSourceIsOneGeneratorRun(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + g := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{g}, nil) + + deps := []Source{ + depSrc("prov-a:v1", "v1", "a.yaml"), + depSrc("prov-b:v1", "v1", "b.yaml"), + depSrc("prov-c:v1", "v1", "c.yaml"), + } + if err := m.GenerateFromMultipleSources(t.Context(), deps, apiSrc("v1", "xrd.yaml")); err != nil { + t.Fatal(err) + } + + if got, want := g.runs, 4; got != want { + t.Errorf("generator ran %d times, want %d (one per source)", got, want) + } + assertFilesPresent(t, testFS, + treeDepPath(0, "prov-a:v1", "a.yaml"), + treeDepPath(1, "prov-b:v1", "b.yaml"), + treeDepPath(2, "prov-c:v1", "c.yaml"), + treeAPIPath(0, "xrd.yaml"), + "mock/shared-root", + ) +} + +// The hidden directories ignore themselves for git, so projects that check +// schemas/ in do not pick up the cache, and staging is gone once a pass ends, +// including a pass that replaced a previous tree. +func TestHiddenDirectoriesStayOutOfGit(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + m := New(testFS, []generator.Interface{&kindGenerator{lang: "mock"}}, nil) + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + for _, xrd := range []string{"v1", "v2"} { + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, apiSrc(xrd, "xrd-"+xrd+".yaml")); err != nil { + t.Fatal(err) + } + if ok, _ := afero.Exists(testFS, stagingDirName); ok { + t.Errorf("after the %s pass: %s left behind", xrd, stagingDirName) + } + } + + bs, err := afero.ReadFile(testFS, path.Join(cacheDirName, ".gitignore")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(ignoreAllGitignore, string(bs)); diff != "" { + t.Errorf("%s/.gitignore (-want +got):\n%s", cacheDirName, diff) + } +} + +func TestJSONBuildIndexSchema(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + reason string + files []string + wantRef []string // nil: no index is written + }{ + "NoSchemas": { + reason: "A tree with no JSON schemas must not get an empty index.", + }, + "Schemas": { + reason: "The index must reference every schema in the tree.", + files: []string{"a.schema.json", "b.schema.json"}, + wantRef: []string{"a.schema.json", "b.schema.json"}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + langFS := afero.NewMemMapFs() + for _, f := range tc.files { + if err := afero.WriteFile(langFS, f, []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := jsonBuildIndexSchema(langFS); err != nil { + t.Fatalf("%s\njsonBuildIndexSchema(...): %v", tc.reason, err) + } + + bs, err := afero.ReadFile(langFS, "index.schema.json") + if tc.wantRef == nil { + if err == nil { + t.Errorf("%s\nwrote an index for no schemas: %s", tc.reason, bs) + } + return + } + if err != nil { + t.Fatalf("%s\nno index written: %v", tc.reason, err) + } + var index struct { + AnyOf []struct { + Ref string `json:"$ref"` + } `json:"anyOf"` + } + if err := json.Unmarshal(bs, &index); err != nil { + t.Fatal(err) + } + var got []string + for _, s := range index.AnyOf { + got = append(got, s.Ref) + } + if diff := cmp.Diff(tc.wantRef, got); diff != "" { + t.Errorf("%s\nindex refs: -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +// A pass checks every generated file against the tree manifest. A generated +// file that was edited or deleted is restored from the group caches without +// regenerating anything; a file added to the tree is left alone, so tools +// writing there (Python's __pycache__) don't cause an assembly every pass. +func TestGeneratedFilesAreChecked(t *testing.T) { + t.Parallel() + + dep := depSrc("prov-a:v1", "v1", "dep.yaml") + depFile := treeDepPath(0, "prov-a:v1", "dep.yaml") + const extra = "mock/__pycache__/dep.cpython-312.pyc" + + cases := map[string]struct { + reason string + change func(t *testing.T, fsys afero.Fs) + // want is the content of depFile after the next pass, "" if absent. + want string + // extraKept is whether the extra file survives the next pass, if + // the change added one. + extraKept bool + }{ + "EditedFileIsRestored": { + reason: "A hand-edited generated file must be restored from the caches.", + change: func(t *testing.T, fsys afero.Fs) { + t.Helper() + if err := afero.WriteFile(fsys, depFile, []byte("edited"), 0o644); err != nil { + t.Fatal(err) + } + }, + want: "dep.yaml", + }, + "DeletedFileIsRestored": { + reason: "A generated file deleted from an otherwise intact tree must be restored.", + change: func(t *testing.T, fsys afero.Fs) { + t.Helper() + if err := fsys.Remove(depFile); err != nil { + t.Fatal(err) + } + }, + want: "dep.yaml", + }, + "AddedFileIsLeftAlone": { + reason: "A file added to the tree is not drift, so the tree is not reassembled over it.", + change: func(t *testing.T, fsys afero.Fs) { + t.Helper() + if err := afero.WriteFile(fsys, extra, []byte("bytecode"), 0o644); err != nil { + t.Fatal(err) + } + }, + want: "dep.yaml", + extraKept: true, + }, + "MissingManifestReassembles": { + reason: "Without a manifest the tree cannot be checked, so it is reassembled, which also clears added files.", + change: func(t *testing.T, fsys afero.Fs) { + t.Helper() + if err := afero.WriteFile(fsys, extra, []byte("bytecode"), 0o644); err != nil { + t.Fatal(err) + } + if err := fsys.Remove(path.Join(cacheDirName, treeManifestFileName)); err != nil { + t.Fatal(err) + } + }, + want: "dep.yaml", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ctx := t.Context() + testFS := afero.NewMemMapFs() + gen := &kindGenerator{lang: "mock"} + m := New(testFS, []generator.Interface{gen}, nil) + api := apiSrc("v1", "xrd.yaml") + + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + runs := gen.runs + + tc.change(t, testFS) + if err := m.GenerateFromMultipleSources(ctx, []Source{dep}, api); err != nil { + t.Fatal(err) + } + + if got := readFile(t, testFS, depFile); got != tc.want { + t.Errorf("%s\n%s = %q, want %q", tc.reason, depFile, got, tc.want) + } + if gen.runs != runs { + t.Errorf("%s\nthe generator ran %d more times; restoring a file must reuse the caches", tc.reason, gen.runs-runs) + } + if ok, _ := afero.Exists(testFS, extra); ok != tc.extraKept { + t.Errorf("%s\n%s present = %v, want %v", tc.reason, extra, ok, tc.extraKept) + } + }) + } } diff --git a/internal/schemas/manager/source.go b/internal/schemas/manager/source.go index feed85a7..a762341f 100644 --- a/internal/schemas/manager/source.go +++ b/internal/schemas/manager/source.go @@ -17,6 +17,8 @@ limitations under the License. package manager import ( + "bufio" + "bytes" "context" "crypto/sha256" "fmt" @@ -36,6 +38,8 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5/storage/memory" "github.com/spf13/afero" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" @@ -67,10 +71,35 @@ type Source interface { Type() SourceType } +// describer is implemented by sources that have a more readable name for +// messages than their ID. IDs are lock keys and carry a scheme prefix +// (http://https://example.com/crds.yaml), which reads badly in an error. +type describer interface { + Description() string +} + +// describeSource returns the name to use for a source in messages. +func describeSource(src Source) string { + if d, ok := src.(describer); ok { + return d.Description() + } + return src.ID() +} + // calculateFilesystemHash calculates a SHA256 hash of the filesystem contents. +// A filesystem whose root does not exist hashes as empty: a source whose +// directory has not been created yet (e.g. a Configuration with no apis/) +// contributes no resources, and must not fail the generation pass. func calculateFilesystemHash(filesystem afero.Fs, sourceType SourceType) (string, error) { h := sha256.New() + if _, err := filesystem.Stat("."); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Sprintf("%x", h.Sum(nil)), nil + } + return "", err + } + var extensions []string switch sourceType { case SourceTypeCRD: @@ -112,33 +141,221 @@ func calculateFilesystemHash(filesystem afero.Fs, sourceType SourceType) (string return fmt.Sprintf("%x", h.Sum(nil)), nil } -// fsSource is a resource source backed by a filesystem. +// fsSource is a resource source backed by a filesystem. Only its XRD and CRD +// documents count: they are what Resources returns and what Version covers, +// so editing a Composition or any other YAML beside the APIs never reads as +// an API change. type fsSource struct { - id string - fs afero.Fs + id string + fs afero.Fs + exclude []string } func (f *fsSource) ID() string { return "fs://" + f.id } -func (f *fsSource) Version(_ context.Context) (string, error) { - return calculateFilesystemHash(f.fs, SourceTypeCRD) +func (f *fsSource) Description() string { + if p := cleanProjectPath(f.id); p != "" { + return p + "/" + } + return "the project root" } +// Version hashes exactly the definitions Resources returns, each with its +// path, so a change anywhere else under the source's root -- a Composition, +// a function's node_modules, the generated schemas -- never reads as an API +// change, while renaming a definition's file does. +func (f *fsSource) Version(ctx context.Context) (string, error) { + res, err := f.Resources(ctx) + if err != nil { + return "", err + } + h := sha256.New() + // afero.Walk visits paths in lexical order, so the hash is stable. + err = afero.Walk(res, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := afero.ReadFile(res, p) + if err != nil { + return err + } + _, _ = h.Write([]byte(filepath.ToSlash(p))) + _, _ = h.Write([]byte{0}) + _, _ = h.Write(content) + _, _ = h.Write([]byte{0}) + return nil + }) + if err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// Resources returns the source's XRD and CRD documents, each file holding the +// definitions it had. Other documents, and files with none, are left out: the +// generators only read definitions, and Version must change exactly when +// their input does. Excluded paths and hidden directories are skipped: a +// project whose APIs live at its root ("/") would otherwise hand the +// generators every file it has, including function dependencies and the +// schemas directory's own caches. func (f *fsSource) Resources(_ context.Context) (afero.Fs, error) { - return f.fs, nil + out := afero.NewMemMapFs() + + // A source whose directory does not exist yet (a project that declares + // apis/ but has not created it) contributes no resources. + if _, err := f.fs.Stat("."); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return out, nil + } + return nil, err + } + + err := afero.Walk(f.fs, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + rel := strings.TrimPrefix(filepath.ToSlash(p), "/") + if info.IsDir() { + if rel != "." && rel != "" && (strings.HasPrefix(info.Name(), ".") || f.excluded(rel)) { + return filepath.SkipDir + } + return nil + } + ext := strings.ToLower(filepath.Ext(p)) + if ext != ".yaml" && ext != ".yml" { + return nil + } + bs, err := afero.ReadFile(f.fs, p) + if err != nil { + return errors.Wrapf(err, "failed to read %s", p) + } + defs, err := definitions(bs) + if err != nil { + return errors.Wrapf(err, "cannot read the API definitions in %s", p) + } + if len(defs) == 0 { + return nil + } + if err := out.MkdirAll(path.Dir(rel), 0o755); err != nil { + return err + } + return afero.WriteFile(out, rel, bytes.Join(defs, []byte("---\n")), 0o644) + }) + if err != nil { + return nil, err + } + return out, nil +} + +// definitions returns the XRD and CRD documents of a YAML file, in order. A +// document that doesn't parse is an error, as it was when the generators +// parsed every file themselves: dropping it would silently drop any API +// definition it holds from the generated schemas. +func definitions(content []byte) ([][]byte, error) { + var defs [][]byte + reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(content))) + for { + doc, err := reader.Read() + if errors.Is(err, io.EOF) { + return defs, nil + } + if err != nil { + return nil, err + } + var res struct { + Kind string `json:"kind"` + } + if err := yaml.Unmarshal(doc, &res); err != nil { + return nil, errors.Wrap(err, "cannot parse it as YAML; fix its syntax so the API definitions in it are generated") + } + if res.Kind == "CompositeResourceDefinition" || res.Kind == "CustomResourceDefinition" { + defs = append(defs, doc) + } + } +} + +// excluded reports whether a slash-separated path relative to the source's +// root is, or is inside, one of the excluded paths. +func (f *fsSource) excluded(rel string) bool { + for _, ex := range f.exclude { + if rel == ex || strings.HasPrefix(rel, ex+"/") { + return true + } + } + return false } func (f *fsSource) Type() SourceType { return SourceTypeCRD } +// FSSourceOption configures a filesystem source. +type FSSourceOption func(*fsSource) + +// WithExcludedPaths skips the given paths, relative to the source's root, and +// everything under them. +func WithExcludedPaths(paths ...string) FSSourceOption { + return func(f *fsSource) { + for _, p := range paths { + p = strings.Trim(path.Clean(filepath.ToSlash(p)), "/") + if p != "" && p != "." { + f.exclude = append(f.exclude, p) + } + } + } +} + // NewFSSource returns a new filesystem-backed resource source. The id should // be a stable, location-independent identifier (e.g. a project-relative path) // since it is persisted in the schema manager's lockfile. -func NewFSSource(id string, fs afero.Fs) Source { - return &fsSource{id: id, fs: fs} +func NewFSSource(id string, fs afero.Fs, opts ...FSSourceOption) Source { + f := &fsSource{id: id, fs: fs} + for _, o := range opts { + o(f) + } + return f +} + +// NewProjectAPIsSource returns the source for a project's own APIs, read from +// paths.apis under projFS. The APIs path may contain the project's other +// directories -- most often with `apis: .`, which makes it the project root. +// Functions, examples, operations, tests and the generated schemas are then +// excluded, so the source reads only the project's own API definitions. +func NewProjectAPIsSource(projFS afero.Fs, paths *v1alpha1.ProjectPaths) Source { + p := paths + if p == nil { + p = &v1alpha1.ProjectPaths{} + p.Default() + } + + apis := cleanProjectPath(p.APIs) + fsys := projFS + if apis != "" { + fsys = afero.NewBasePathFs(projFS, apis) + } + + var exclude []string + for _, other := range []string{p.Functions, p.Examples, p.Operations, p.Tests, p.Schemas} { + o := cleanProjectPath(other) + switch { + case o == "" || o == apis: + // Unset, or the same directory as the APIs: nothing to exclude. + case apis == "": + exclude = append(exclude, o) + case strings.HasPrefix(o, apis+"/"): + exclude = append(exclude, strings.TrimPrefix(o, apis+"/")) + } + } + + return NewFSSource(p.APIs, fsys, WithExcludedPaths(exclude...)) +} + +// cleanProjectPath normalizes a project-relative path to slash-separated form +// with no leading or trailing slash. The project root is "". +func cleanProjectPath(p string) string { + return strings.Trim(path.Clean("/"+filepath.ToSlash(p)), "/") } // xpkgSource is a source backed by extracted CRDs from an xpkg. Unlike the @@ -153,6 +370,10 @@ func (s *xpkgSource) ID() string { return "xpkg://" + s.id } +func (s *xpkgSource) Description() string { + return s.id +} + func (s *xpkgSource) Version(_ context.Context) (string, error) { return s.version, nil } @@ -195,6 +416,13 @@ func (g *gitSource) ID() string { return id } +func (g *gitSource) Description() string { + if g.gitRef.Path != "" { + return g.gitRef.Repository + " at " + g.gitRef.Path + } + return g.gitRef.Repository +} + func (g *gitSource) Version(ctx context.Context) (string, error) { if g.commitSHA != "" { return g.commitSHA, nil @@ -330,6 +558,10 @@ func (h *httpSource) ID() string { return fmt.Sprintf("http://%s", h.httpRef.URL) } +func (h *httpSource) Description() string { + return h.httpRef.URL +} + func (h *httpSource) Version(ctx context.Context) (string, error) { if h.fs != nil { return h.calculateHash() @@ -444,6 +676,10 @@ func (k *k8sOpenAPISource) ID() string { return fmt.Sprintf("k8s://%s", k.k8sRef.Version) } +func (k *k8sOpenAPISource) Description() string { + return "Kubernetes API " + k.k8sRef.Version +} + func (k *k8sOpenAPISource) Version(_ context.Context) (string, error) { return k.k8sRef.Version, nil } @@ -466,8 +702,11 @@ func (k *k8sOpenAPISource) Resources(ctx context.Context) (afero.Fs, error) { } defer resp.Body.Close() //nolint:errcheck // nothing to do + if resp.StatusCode == http.StatusNotFound { + return nil, errors.Errorf("no Kubernetes OpenAPI spec found for version %s (HTTP 404 from %s); check that this Kubernetes version exists", k.k8sRef.Version, specURL) + } if resp.StatusCode != http.StatusOK { - return nil, errors.Errorf("failed to download Kubernetes OpenAPI spec: HTTP %d", resp.StatusCode) + return nil, errors.Errorf("failed to download the Kubernetes OpenAPI spec for version %s: HTTP %d from %s", k.k8sRef.Version, resp.StatusCode, specURL) } content, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPSize)) diff --git a/internal/schemas/manager/source_test.go b/internal/schemas/manager/source_test.go index 387cfe75..0407a63c 100644 --- a/internal/schemas/manager/source_test.go +++ b/internal/schemas/manager/source_test.go @@ -17,6 +17,10 @@ limitations under the License. package manager import ( + "context" + "os" + "path/filepath" + "strings" "testing" "github.com/go-git/go-billy/v5" @@ -680,3 +684,275 @@ func TestCalculateFilesystemHashDifferentContent(t *testing.T) { }) } } + +// A filesystem source whose directory does not exist yet must behave as an +// empty source, not an error: projects declare apis/ before creating it. +func TestFSSourceMissingDirectory(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + src := NewFSSource("apis", afero.NewBasePathFs(projFS, "apis")) + + if _, err := src.Version(context.Background()); err != nil { + t.Errorf("Version: %v", err) + } + + resources, err := src.Resources(context.Background()) + if err != nil { + t.Fatalf("Resources: %v", err) + } + empty, err := afero.IsEmpty(resources, ".") + if err != nil { + t.Fatalf("checking resources emptiness: %v", err) + } + if !empty { + t.Error("resources should be empty for a missing directory") + } + + // A missing directory hashes the same as an existing empty one, so + // creating the directory alone never triggers regeneration. + emptyFS := afero.NewMemMapFs() + if err := emptyFS.MkdirAll("apis", 0o755); err != nil { + t.Fatal(err) + } + hashMissing, err := src.Version(context.Background()) + if err != nil { + t.Fatal(err) + } + hashEmpty, err := NewFSSource("apis", afero.NewBasePathFs(emptyFS, "apis")).Version(context.Background()) + if err != nil { + t.Fatal(err) + } + if hashMissing != hashEmpty { + t.Errorf("missing directory hash %q != empty directory hash %q", hashMissing, hashEmpty) + } +} + +// A project whose APIs live at its root (`apis: .`) must hand the generators +// only its own API definitions: not the other project directories, the +// generated schemas, hidden directories such as .git, or non-definition YAML +// such as Compositions. Changes there must not read as an API change either. +func TestProjectAPIsSourceAtRoot(t *testing.T) { + for _, apis := range []string{".", "/", "./"} { + t.Run(apis, func(t *testing.T) { + testProjectAPIsSourceAtRoot(t, apis) + }) + } +} + +func testProjectAPIsSourceAtRoot(t *testing.T, apis string) { + t.Helper() + ctx := context.Background() + projFS := afero.NewMemMapFs() + files := map[string]string{ + "xrd.yaml": "kind: CompositeResourceDefinition", + "nested/composition.yml": "kind: Composition", + "README.md": "not yaml", + "functions/fn/node_modules/pkg/a.yaml": "dependency", + "examples/xr.yaml": "example", + "operations/op.yaml": "operation", + "tests/test.yaml": "test", + "schemas/.cache/apis/json/x.schema.yaml": "generated", + "schemas/python/models/pyproject.yaml": "generated", + ".github/workflows/ci.yaml": "workflow", + "functions-and-more/kept.yaml": "kind: CustomResourceDefinition # a sibling that only shares a prefix", + } + for p, c := range files { + if err := afero.WriteFile(projFS, p, []byte(c), 0o644); err != nil { + t.Fatal(err) + } + } + + paths := &v1alpha1.ProjectPaths{APIs: apis} + paths.Default() + src := NewProjectAPIsSource(projFS, paths) + + res, err := src.Resources(ctx) + if err != nil { + t.Fatal(err) + } + var got []string + if err := afero.Walk(res, ".", func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + got = append(got, filepath.ToSlash(p)) + } + return nil + }); err != nil { + t.Fatal(err) + } + want := []string{"functions-and-more/kept.yaml", "xrd.yaml"} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("resources (-want +got):\n%s", diff) + } + + before, err := src.Version(ctx) + if err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "schemas/.cache/apis/json/x.schema.yaml", []byte("regenerated"), 0o644); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "functions/fn/node_modules/pkg/b.yaml", []byte("new dependency"), 0o644); err != nil { + t.Fatal(err) + } + after, err := src.Version(ctx) + if err != nil { + t.Fatal(err) + } + if before != after { + t.Error("a change outside the project's APIs changed the APIs source version") + } + + if err := afero.WriteFile(projFS, "xrd.yaml", []byte("kind: CompositeResourceDefinition # edited"), 0o644); err != nil { + t.Fatal(err) + } + edited, err := src.Version(ctx) + if err != nil { + t.Fatal(err) + } + if edited == after { + t.Error("an XRD edit did not change the APIs source version") + } +} + +// A project with its own apis/ directory reads only that directory, and a +// directory that does not exist yet is an empty source. +func TestProjectAPIsSourceSubdir(t *testing.T) { + ctx := context.Background() + projFS := afero.NewMemMapFs() + + paths := &v1alpha1.ProjectPaths{} + paths.Default() + src := NewProjectAPIsSource(projFS, paths) + if got, want := src.ID(), "fs://apis"; got != want { + t.Errorf("ID() = %q, want %q", got, want) + } + + empty, err := src.Version(ctx) + if err != nil { + t.Fatalf("missing apis/ directory: %v", err) + } + + if err := afero.WriteFile(projFS, "apis/xrd.yaml", []byte("kind: CompositeResourceDefinition"), 0o644); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "examples/xr.yaml", []byte("example"), 0o644); err != nil { + t.Fatal(err) + } + res, err := src.Resources(ctx) + if err != nil { + t.Fatal(err) + } + if ok, _ := afero.Exists(res, "xrd.yaml"); !ok { + t.Error("xrd.yaml is missing from the APIs source") + } + if ok, _ := afero.Exists(res, "examples/xr.yaml"); ok { + t.Error("the APIs source read outside apis/") + } + if v, _ := src.Version(ctx); v == empty { + t.Error("adding an XRD did not change the APIs source version") + } +} + +// Project directories nested inside the APIs directory are excluded too. +func TestProjectAPIsSourceNestedDirs(t *testing.T) { + ctx := context.Background() + projFS := afero.NewMemMapFs() + for p, c := range map[string]string{ + "src/xrd.yaml": "kind: CompositeResourceDefinition", + "src/examples/xr.yaml": "example", + "src/fns/fn/a.yaml": "function", + "examples-top/other.yaml": "outside src, never read", + } { + if err := afero.WriteFile(projFS, p, []byte(c), 0o644); err != nil { + t.Fatal(err) + } + } + + paths := &v1alpha1.ProjectPaths{APIs: "src", Examples: "src/examples", Functions: "./src/fns/"} + paths.Default() + res, err := NewProjectAPIsSource(projFS, paths).Resources(ctx) + if err != nil { + t.Fatal(err) + } + for p, want := range map[string]bool{"xrd.yaml": true, "examples/xr.yaml": false, "fns/fn/a.yaml": false} { + if got, _ := afero.Exists(res, p); got != want { + t.Errorf("%s present = %v, want %v", p, got, want) + } + } +} + +// Only XRD and CRD documents are API definitions: editing a Composition, +// even one in the same file as an XRD, must not read as an API change, and +// the generators must only see the definitions. +func TestFSSourceDefinitionsOnly(t *testing.T) { + ctx := context.Background() + + const xrd = "apiVersion: apiextensions.crossplane.io/v2\nkind: CompositeResourceDefinition\nmetadata:\n name: xthings.example.org\n" + const comp = "apiVersion: apiextensions.crossplane.io/v1\nkind: Composition\nmetadata:\n name: thing\n" + write := func(t *testing.T, fsys afero.Fs, p, content string) { + t.Helper() + if err := afero.WriteFile(fsys, p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + + fsys := afero.NewMemMapFs() + write(t, fsys, "both.yaml", xrd+"---\n"+comp) + write(t, fsys, "composition.yaml", comp) + src := NewFSSource("apis", fsys) + + res, err := src.Resources(ctx) + if err != nil { + t.Fatal(err) + } + got, err := afero.ReadFile(res, "both.yaml") + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(xrd, string(got)); diff != "" { + t.Errorf("both.yaml: the generators must only see its XRD (-want +got):\n%s", diff) + } + if ok, _ := afero.Exists(res, "composition.yaml"); ok { + t.Error("a file with no definitions must be left out") + } + + before, err := src.Version(ctx) + if err != nil { + t.Fatal(err) + } + write(t, fsys, "both.yaml", xrd+"---\n"+comp+"spec:\n edited: true\n") + write(t, fsys, "composition.yaml", comp+"spec:\n edited: true\n") + if after, _ := src.Version(ctx); after != before { + t.Error("editing Compositions changed the version") + } + + write(t, fsys, "renamed.yaml", xrd) + if err := fsys.Remove("both.yaml"); err != nil { + t.Fatal(err) + } + if renamed, _ := src.Version(ctx); renamed == before { + t.Error("moving an XRD to another file did not change the version") + } +} + +// A YAML syntax error in the project's APIs must fail, naming the file, rather +// than silently dropping the definitions in it from the generated schemas. +func TestFSSourceBrokenYAMLFails(t *testing.T) { + fsys := afero.NewMemMapFs() + if err := afero.WriteFile(fsys, "xrd.yaml", []byte("kind: CompositeResourceDefinition\nspec: [unclosed\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := NewFSSource("apis", fsys).Resources(context.Background()) + if err == nil { + t.Fatal("Resources accepted a file that isn't valid YAML") + } + for _, want := range []string{"xrd.yaml", "fix its syntax"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +}