diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c82fc84c1e..4c7f65f05e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,9 @@ updates: directory: '/' schedule: interval: 'weekly' + commit-message: + prefix: 'chore' + include: 'scope' groups: # Storybook add-ons declare a peer dependency on the matching `storybook` # release, so they have to move together. Bump them in a single PR. @@ -32,3 +35,6 @@ updates: directory: '/' schedule: interval: 'weekly' + commit-message: + prefix: 'chore' + include: 'scope' diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000000..e1df0c159d --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,37 @@ +# Groups the notes that GitHub generates for each release (see +# `generate_release_notes` in workflows/release.yaml) by PR label. +# Labels are applied automatically by workflows/pr-labels.yaml. +# A PR is listed under the first category whose labels it matches. +# RELEASING.md refers to these section titles, so keep the two in sync. +changelog: + exclude: + labels: + - skip-changelog + - ci + - test + - dependencies + categories: + - title: Breaking changes + labels: + - breaking-change + - title: State migrations (run `sqlmesh migrate` after upgrading) + labels: + - migration + - title: Features + labels: + - feature + - title: Performance + labels: + - performance + - title: Bug fixes + labels: + - bug + - title: Documentation + labels: + - documentation + - title: Maintenance + labels: + - maintenance + - title: Other changes + labels: + - "*" diff --git a/.github/workflows/pr-labels.yaml b/.github/workflows/pr-labels.yaml new file mode 100644 index 0000000000..b7a000d10b --- /dev/null +++ b/.github/workflows/pr-labels.yaml @@ -0,0 +1,161 @@ +name: PR Labels +# Checks the PR title format and applies the labels that .github/release.yml +# uses to group release notes. +# +# pull_request_target is used so that labels can be applied to PRs from forks. +# This workflow never checks out or runs code from the PR. +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + contents: read + issues: write + pull-requests: write + +concurrency: + group: pr-labels-${{ github.event.pull_request.number }} + # Queue rather than cancel, so a run is never stopped part-way through + # changing labels. + cancel-in-progress: false + +jobs: + title: + name: Check title and label by type + # Runs on pushes too: GitHub keeps only one pending run per concurrency + # group, so a push can replace a queued run for a title edit. + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v8 + env: + PR_TITLE: ${{ github.event.pull_request.title }} + with: + script: | + const title = process.env.PR_TITLE; + const issue_number = context.payload.pull_request.number; + const { owner, repo } = context.repo; + + const typeLabels = new Map([ + ['feat', 'feature'], + ['fix', 'bug'], + ['perf', 'performance'], + ['docs', 'documentation'], + ['refactor', 'maintenance'], + ['chore', 'maintenance'], + ['build', 'maintenance'], + ['revert', null], + ['ci', 'ci'], + ['test', 'test'], + ]); + // Scopes for internal-only work, which release notes leave out + // (e.g. "fix(ci): ..." is labelled ci, not bug). + const scopeLabels = new Map([['ci', 'ci'], ['test', 'test'], ['tests', 'test']]); + // Labels this job owns. The title is the source of truth for them: + // any that don't match the current title are removed. + const colors = new Map([ + ['breaking-change', 'b60205'], + ['feature', '0e8a16'], + ['bug', 'd73a4a'], + ['performance', 'fbca04'], + ['documentation', '0075ca'], + ['maintenance', 'c5def5'], + ['ci', 'ededed'], + ['test', 'ededed'], + ]); + + let wanted; + if (/^Revert "/.test(title)) { + // Title generated by GitHub's "Revert" button. + wanted = new Set(); + } else { + const m = /^([a-z]+)(?:\(([^)]+)\))?(!)?: \S/.exec(title); + if (!m || !typeLabels.has(m[1])) { + core.setFailed( + `PR title "${title}" must look like "type(scope): description" or ` + + `"type: description", where type is lowercase and one of: ` + + `${[...typeLabels.keys()].join(', ')}. Add "!" after the type or scope ` + + `for a breaking change, e.g. "feat(plan)!: ...". See CONTRIBUTING.md. ` + + `Re-running this job checks the old title again: edit the title instead.` + ); + return; + } + const [, type, scope, bang] = m; + wanted = new Set(); + if (bang) { + // Breaking changes always appear in the release notes, so they + // never get the ci/test labels that exclude a PR. + wanted.add('breaking-change'); + const label = typeLabels.get(type); + if (label && label !== 'ci' && label !== 'test') wanted.add(label); + } else { + const label = (scope && scopeLabels.get(scope.toLowerCase())) || typeLabels.get(type); + if (label) wanted.add(label); + } + } + + const { data: live } = await github.rest.issues.listLabelsOnIssue({ + owner, repo, issue_number, per_page: 100, + }); + const current = live.map((l) => l.name); + + for (const name of current) { + if (!colors.has(name) || wanted.has(name)) continue; + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number, name }); + } catch (e) { + if (e.status !== 404) throw e; + } + } + + const toAdd = [...wanted].filter((name) => !current.includes(name)); + for (const name of toAdd) { + try { + await github.rest.issues.getLabel({ owner, repo, name }); + } catch (e) { + if (e.status !== 404) throw e; + await github.rest.issues.createLabel({ owner, repo, name, color: colors.get(name) }); + } + } + if (toAdd.length) { + await github.rest.issues.addLabels({ owner, repo, issue_number, labels: toAdd }); + } + + migrations: + name: Label new state migrations + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v8 + with: + script: | + const issue_number = context.payload.pull_request.number; + const { owner, repo } = context.repo; + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, repo, pull_number: issue_number, per_page: 100, + }); + const added = files.some( + (f) => f.status === 'added' && /^sqlmesh\/migrations\/v\d+_.*\.py$/.test(f.filename) + ); + // Read live labels: a queued run's event payload can be out of date. + const { data: live } = await github.rest.issues.listLabelsOnIssue({ + owner, repo, issue_number, per_page: 100, + }); + const labels = live.map((l) => l.name); + if (!added) { + // The PR no longer adds a migration, e.g. the file was removed. + if (labels.includes('migration')) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number, name: 'migration' }); + } catch (e) { + if (e.status !== 404) throw e; + } + } + return; + } + if (labels.includes('migration')) return; + try { + await github.rest.issues.getLabel({ owner, repo, name: 'migration' }); + } catch (e) { + if (e.status !== 404) throw e; + await github.rest.issues.createLabel({ owner, repo, name: 'migration', color: '5319e7' }); + } + await github.rest.issues.addLabels({ owner, repo, issue_number, labels: ['migration'] }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4533069d5c..3041922db9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,34 @@ Optionally, `make install-pre-commit` installs git hooks so ruff and mypy run on - Include a DCO sign-off on all commits (`git commit -s`) - Be responsive to review feedback from maintainers +### Pull Request Titles + +Pull requests are squash-merged, so the title becomes the commit message and the +entry in the release notes. Titles must follow this format, which CI checks: + +``` +type(scope): description +``` + +- `type` is one of `feat`, `fix`, `perf`, `docs`, `refactor`, `chore`, `build`, `revert`, `ci` or `test`. +- `scope` is optional and names the area changed, e.g. `snowflake`, `plan`, `dbt` or `vscode`. +- Add `!` after the type or scope for a breaking change, e.g. `feat(plan)!: drop support for X`. +- Write the description for users: say what changed for them, not how the code changed. + +Titles created by GitHub's "Revert" button (`Revert "..."`) are also accepted. If the +title check fails, edit the title: re-running the failed job checks the old title again. + +Release notes are grouped using labels that are applied automatically from the title +and the files changed. The title decides the `feature`, `bug`, `performance`, +`documentation`, `maintenance`, `ci`, `test` and `breaking-change` labels, so to +recategorize a PR, change its title rather than its labels. Use `ci` or `test` as the type +or scope for internal-only changes (e.g. `fix(ci): ...`) so they are left out of the +release notes. A maintainer can add the `skip-changelog` label to leave out any other PR. + +## Releases + +Maintainers publish releases. See [RELEASING.md](RELEASING.md) for the steps. + ## Licensing Code contributions are licensed under the [Apache License 2.0](LICENSE). Documentation contributions are licensed under [Creative Commons Attribution 4.0 International (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/). See the LICENSE file and the [technical charter](sqlmesh-technical-charter.pdf) for details. diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000000..5b92e257f5 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,76 @@ +# Releasing SQLMesh + +This guide is for maintainers with write access to `SQLMesh/sqlmesh`. It covers the three things the project publishes: + +| Package | Registry | How a release starts | +|---|---|---| +| `sqlmesh` (Python) | PyPI | Push a `v*.*.*` tag | +| SQLMesh VS Code extension | VS Code Marketplace and Open VSX | Run the **Release VSCode Extension** workflow | +| `@tobikodata/sqlmesh-common` | npm | Run the **Release web common code** workflow | + +## Release the Python Package + +The [`Release`](.github/workflows/release.yaml) workflow runs when a tag that matches `v*.*.*` is pushed. The package version comes from the tag through `setuptools_scm`, so you don't edit a version number in the code. + +The workflow does four things: + +1. Builds the web UI and bundles it into the package. +2. Publishes `sqlmesh` to PyPI with `make publish`. +3. Publishes the test package to the private `tobiko-private` registry with `make publish-tests`. +4. Creates a GitHub release with notes generated from the pull requests merged since the previous tag. + +Each step publishes immediately and there's no approval gate. Check `main` before you tag. + +### Steps + +1. Pick the commit to release. It must be on `main`, and every required check must pass on it. Check the latest **pr.yaml** run for that commit on the [Actions page](https://github.com/SQLMesh/sqlmesh/actions/workflows/pr.yaml). +2. Pick the version. Fetch the tags, then find the highest released version and increment it: + + ```bash + git fetch main --tags + git tag --list 'v[0-9]*' --sort=-v:refname | head -1 + ``` + + Replace `` with your remote for `SQLMesh/sqlmesh`. + +3. Tag the commit and push the tag. Set `VERSION` to the version from step 2, for example `v0.236.5`: + + ```bash + VERSION=v0.236.5 + git tag "$VERSION" + git push "$VERSION" + ``` + + Push only the tag, not a branch. + +4. Watch the [Release](https://github.com/SQLMesh/sqlmesh/actions/workflows/release.yaml) run until it finishes. +5. Check that the new version is on [PyPI](https://pypi.org/project/sqlmesh/#history). +6. Edit the generated release on the [Releases page](https://github.com/SQLMesh/sqlmesh/releases). Under **Breaking changes**, add a short upgrade note for each entry. Say what users will see after upgrading, such as a migration or a metadata-only change in `sqlmesh plan`. + +If the workflow fails after the PyPI step, don't delete or reuse the tag. PyPI doesn't accept the same version twice. Fix the cause and release the next version. + +### Release Notes + +GitHub builds the notes from merged pull request titles and groups them by label, as configured in [`.github/release.yml`](.github/release.yml). The [PR Labels](.github/workflows/pr-labels.yaml) workflow applies the labels from each pull request's title (`type(scope): description`), and adds `migration` to pull requests that add a state migration. See [Pull Request Titles](CONTRIBUTING.md#pull-request-titles) in `CONTRIBUTING.md`. + +- A `!` in the title, such as `fix!:` or `feat!:`, puts the pull request under **Breaking changes**. +- A pull request that adds a state migration goes under **State migrations**. +- CI, test and Dependabot pull requests are left out. Add the `skip-changelog` label to leave out any other pull request. + +Each pull request appears in only one section: the first one in this list that matches its labels. A breaking change that also adds a state migration is listed under **Breaking changes**, so its upgrade note in step 6 should say to run `sqlmesh migrate`. + +To move a pull request to another section, change its title. The workflow resets the type labels from the title. + +## Release the VS Code Extension + +The [Release VSCode Extension](.github/workflows/release_extension.yaml) workflow publishes to the VS Code Marketplace and to Open VSX. + +1. On the workflow's page, choose **Run workflow**. +2. Select the `main` branch. The workflow stops if you run it from any other branch. +3. Enter the version without a `v`, for example `0.0.8`. + +The workflow sets the version only in its own build. It doesn't commit it, so `vscode/extension/package.json` keeps its old version. Check the [Marketplace listing](https://marketplace.visualstudio.com/items?itemName=tobikodata.sqlmesh) for the latest published version and enter a higher one. + +## Release the Shared Web Package + +The [Release web common code](.github/workflows/release_shared_js.yaml) workflow publishes `@tobikodata/sqlmesh-common` from `web/common` to npm. Run it the same way as the extension workflow: from `main`, with a version that has no `v`. It also doesn't commit the version, so check [npm](https://www.npmjs.com/package/@tobikodata/sqlmesh-common) for the latest published version first.