Generate schemas in one merged, cached pass

cmd/crossplane/composition/generate.go

@@ -19,6 +19,7 @@ package composition
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -121,9 +122,15 @@ func (c *generateCmd) AfterApply(cfg *config.Config) error {
 func (c *generateCmd) Run(sp terminal.SpinnerPrinter) error {
 	ctx := context.Background()
 
-	if err := sp.WrapWithSuccessSpinner("Ensuring function-auto-ready dependency", func() error {
+	err := sp.WrapWithSuccessSpinner("Ensuring function-auto-ready dependency", func() error {
 		return c.ensureFunctionAutoReady(ctx)
-	}); err != nil {
+	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	if c.depManager != nil {
+		c.depManager.PrintWarnings(os.Stderr)
+	}
+	if err != nil {
 		return errors.Wrap(err, "failed to ensure function-auto-ready dependency")
 	}
 

cmd/crossplane/dependency/add.go

@@ -18,6 +18,7 @@ package dependency
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -108,9 +109,13 @@ func (c *addCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con
 
 	desc := dependency.GetSourceDescription(dep)
 	logger.Debug("Adding dependency", "dependency", desc)
-	return sp.WrapWithSuccessSpinner("Adding "+desc, func() error {
+	err = sp.WrapWithSuccessSpinner("Adding "+desc, func() error {
 		return m.AddDependency(ctx, &dep)
 	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	m.PrintWarnings(os.Stderr)
+	return err
 }
 
 func (c *addCmd) buildDependency() (v1alpha1.Dependency, error) {

cmd/crossplane/dependency/cache.go

@@ -19,6 +19,7 @@ package dependency
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/alecthomas/kong"
@@ -106,9 +107,13 @@ func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter,
 	m := dependency.NewManager(proj, projFS, opts...)
 
 	logger.Debug("Updating all dependencies")
-	return sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
+	err = sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
 		return m.RefreshAll(ctx, ch)
 	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	m.PrintWarnings(os.Stderr)
+	return err
 }
 
 //go:embed help/clean-cache.md

cmd/crossplane/function/generate.go

@@ -23,6 +23,7 @@ import (
 	"embed"
 	"fmt"
 	"io/fs"
+	"os"
 	"path"
 	"path/filepath"
 	"slices"
@@ -40,13 +41,15 @@ import (
 
 	v1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1"
 	"github.com/crossplane/cli/v2/internal/config"
+	"github.com/crossplane/cli/v2/internal/dependency"
 	"github.com/crossplane/cli/v2/internal/filesystem"
 	"github.com/crossplane/cli/v2/internal/kcl"
 	"github.com/crossplane/cli/v2/internal/project/projectfile"
 	"github.com/crossplane/cli/v2/internal/schemas/generator"
 	"github.com/crossplane/cli/v2/internal/schemas/manager"
 	"github.com/crossplane/cli/v2/internal/schemas/runner"
 	"github.com/crossplane/cli/v2/internal/terminal"
+	clixpkg "github.com/crossplane/cli/v2/internal/xpkg"
 )
 
 //go:embed help/generate.md
@@ -117,9 +120,39 @@ func (c *generateCmd) AfterApply() error {
 	c.fsPath = path.Join(proj.Spec.Paths.Functions, c.Name)
 	c.projectRepository = proj.Spec.Repository
 	c.projectSource = proj.Spec.Source
+
 	return nil
 }
 
+// newDependencyManager builds the dependency manager used to generate
+// schemas in Run, mirroring the project build command's wiring.
+func (c *generateCmd) newDependencyManager(cfg *config.Config) (*dependency.Manager, error) {
+	cacheDir := dependency.DefaultCacheDir()
+	schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas)
+	generators := generator.Filter(
+		generator.AllLanguages(
+			generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors),
+			generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects),
+		),
+		c.proj.Spec.Schemas.GetLanguages(),
+	)
+	client, err := clixpkg.NewClient(
+		clixpkg.NewRemoteFetcher(),
+		clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir),
+		clixpkg.WithImageConfigs(c.proj.Spec.ImageConfigs),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot set up the package client that fetches the project's dependencies")
+	}
+	return dependency.NewManager(c.proj, c.projFS,
+		dependency.WithProjectFile(filepath.Base(c.ProjectFile)),
+		dependency.WithSchemaFS(schemasFS),
+		dependency.WithSchemaGenerators(generators),
+		dependency.WithXpkgClient(client),
+		dependency.WithResolver(clixpkg.NewResolver(client)),
+	), nil
+}
+
 // validateLanguageAgainstSchemas refuses to generate a function in a language
 // whose schemas the project doesn't generate. Such a function would have no
 // models to import, which is surprising, so we fail up front rather than
@@ -154,10 +187,6 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error
 	}
 
 	ctx := context.Background()
-	apisFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.APIs)
-	if c.proj.Spec.Paths.APIs == "/" {
-		apisFS = c.projFS
-	}
 	schemaMgr := manager.New(
 		c.schemasFS,
 		generator.Filter(
@@ -170,10 +199,31 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error
 		runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)),
 	)
 
-	if err := sp.WrapWithSuccessSpinner("Generating schemas", func() error {
-		_, err := schemaMgr.Generate(ctx, manager.NewFSSource(c.proj.Spec.Paths.APIs, apisFS))
-		return err
-	}); err != nil {
+	var depMgr *dependency.Manager
+	err := sp.WrapWithSuccessSpinner("Generating schemas", func() error {
+		var err error
+		depMgr, err = c.newDependencyManager(cfg)
+		if err != nil {
+			return err
+		}
+
+		// Unlike before this PR, generating a function now collects every
+		// dependency declared in the project file, like project build does:
+		// the generated function's models need the dependencies, not just
+		// apis/. The first run in a project needs network access (or a warm
+		// per-user package cache); later runs reuse the project's caches.
+		depSources, err := depMgr.CollectSources(ctx, nil)
+		if err != nil {
+			return errors.Wrap(err, "cannot fetch the project's dependencies, whose models the function needs; check network access and registry credentials")
+		}
+		return schemaMgr.GenerateFromMultipleSources(ctx, depSources, manager.NewProjectAPIsSource(c.projFS, c.proj.Spec.Paths))
+	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	if depMgr != nil {
+		depMgr.PrintWarnings(os.Stderr)
+	}
+	if err != nil {
 		return errors.Wrap(err, "failed to generate schemas")
 	}
 

cmd/crossplane/function/help/generate.md

@@ -3,6 +3,12 @@ language under the project's `functions/` directory. It optionally idempotently
 adds the new function to the end of a Composition's pipeline when given a
 Composition path.
 
+Before it writes the function, the command generates schemas for the
+project's own APIs and for every dependency in the project file, so the
+function can use their models. Dependencies that aren't in the per-user
+package cache are fetched, so the first run in a project may need network
+access and registry credentials.
+
 ## Supported languages
 
 The following are valid arguments to the `--language` / `-l` flag:

cmd/crossplane/project/build.go

@@ -158,6 +158,9 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *c
 		)
 		return buildErr
 	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	depMgr.PrintWarnings(os.Stderr)
 	if err != nil {
 		return err
 	}

cmd/crossplane/project/run.go

@@ -253,6 +253,9 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con
 
 		return eg.Wait()
 	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	depMgr.PrintWarnings(os.Stderr)
 	if err != nil {
 		return err
 	}

cmd/crossplane/render/op/cmd.go

@@ -383,7 +383,7 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal
 		return nil, errors.Wrap(err, "cannot load project functions")
 	}
 
-	if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
+	buildErr := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
 		schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas)
 		schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs))
 		schemaMgr := manager.New(schemasFS, generators, schemaRunner)
@@ -417,8 +417,12 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal
 		embeddedFns, err := project.EmbeddedFunctionsToDaemon(ctx, imgMap)
 		fns = append(fns, embeddedFns...)
 		return err
-	}); err != nil {
-		return nil, errors.Wrap(err, "cannot build embedded functions")
+	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	depMgr.PrintWarnings(os.Stderr)
+	if buildErr != nil {
+		return nil, errors.Wrap(buildErr, "cannot build embedded functions")
 	}
 
 	return fns, nil

cmd/crossplane/render/xr/cmd.go

@@ -495,7 +495,7 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger,
 		return nil, errors.Wrap(err, "cannot load project functions")
 	}
 
-	if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
+	buildErr := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error {
 		schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas)
 		schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs))
 		schemaMgr := manager.New(schemasFS, generators, schemaRunner)
@@ -526,8 +526,12 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger,
 		embeddedFns, err := project.EmbeddedFunctionsToDaemon(ctx, imgMap)
 		fns = append(fns, embeddedFns...)
 		return err
-	}); err != nil {
-		return nil, errors.Wrap(err, "cannot build embedded functions")
+	})
+	// Collection warnings are printed once the spinner has stopped, failed or
+	// not: a warning can explain the failure.
+	depMgr.PrintWarnings(os.Stderr)
+	if buildErr != nil {
+		return nil, errors.Wrap(buildErr, "cannot build embedded functions")
 	}
 
 	return fns, nil