diff --git a/sqlmesh/core/audit/definition.py b/sqlmesh/core/audit/definition.py index 4c90151ee4..81e3830420 100644 --- a/sqlmesh/core/audit/definition.py +++ b/sqlmesh/core/audit/definition.py @@ -291,6 +291,16 @@ def metadata_hash(self) -> str: data.append(self.query_.sql) data.extend([e.sql for e in self.expressions_ or []]) + + # Only include the registry when it holds macros, so that audits without Jinja + # macros keep their existing hash. The registry is trimmed to the referenced macros. + # The refs / sources / vars from `global_objs` are therefore hashed only when macros + # are present, which matches how models hash their registry. Trimming can leave empty + # packages behind (for example `target` from `{{ target.name }}`), so only packages that + # hold macros count. + if self.jinja_macros.root_macros or any(self.jinja_macros.packages.values()): + data.extend(self.jinja_macros.data_hash_values) + self._metadata_hash = hash_data(data) return self._metadata_hash diff --git a/sqlmesh/migrations/v0103_include_jinja_macros_in_standalone_audit_fingerprint.py b/sqlmesh/migrations/v0103_include_jinja_macros_in_standalone_audit_fingerprint.py new file mode 100644 index 0000000000..034f1db110 --- /dev/null +++ b/sqlmesh/migrations/v0103_include_jinja_macros_in_standalone_audit_fingerprint.py @@ -0,0 +1,9 @@ +"""Include referenced Jinja macros in the standalone audit fingerprint.""" + + +def migrate_schemas(engine_adapter, schema, **kwargs): # type: ignore + pass + + +def migrate_rows(engine_adapter, schema, **kwargs): # type: ignore + pass diff --git a/sqlmesh/utils/jinja.py b/sqlmesh/utils/jinja.py index 829981db1c..b50b7cd3da 100644 --- a/sqlmesh/utils/jinja.py +++ b/sqlmesh/utils/jinja.py @@ -529,7 +529,11 @@ def data_hash_values(self) -> t.List[str]: trimmed_global_objs = { k: self.global_objs[k] for k in ("refs", "sources", "vars") if k in self.global_objs } - data.append(json.dumps(trimmed_global_objs, sort_keys=True)) + # `default=str` covers values that JSON can't represent, such as dbt vars loaded from an + # unquoted YAML date. It doesn't apply to JSON-serializable values, so their hashes are + # unchanged. A snapshot stored in state keeps such a value as its string, so hashing the + # string form keeps the hash the same after the snapshot is loaded back. + data.append(json.dumps(trimmed_global_objs, sort_keys=True, default=str)) return data diff --git a/tests/core/test_audit.py b/tests/core/test_audit.py index ac2e8abe16..baa7200fc7 100644 --- a/tests/core/test_audit.py +++ b/tests/core/test_audit.py @@ -1,4 +1,5 @@ import json +import typing as t import pytest from sqlglot import exp, parse_one @@ -1233,3 +1234,143 @@ def test_audit_formatting_flag_serde(): deserialized_audit = ModelAudit.parse_raw(audit_json) assert deserialized_audit.dict() == audit.dict() + + +def _load_standalone_audit_with_macros( + macros: str, + query_body: str, + package_macros: t.Optional[t.Dict[str, str]] = None, + global_objs: t.Optional[t.Dict[str, t.Any]] = None, +) -> StandaloneAudit: + expressions = parse( + f""" + AUDIT ( + name my_audit, + standalone true, + ); + + JINJA_QUERY_BEGIN; + {query_body} + JINJA_END; + """ + ) + jinja_macros = JinjaMacroRegistry(global_objs=global_objs or {}) + jinja_macros.add_macros(MacroExtractor().extract(macros)) + for package, package_source in (package_macros or {}).items(): + jinja_macros.add_macros(MacroExtractor().extract(package_source), package=package) + audit = load_audit(expressions, jinja_macros=jinja_macros) + assert isinstance(audit, StandaloneAudit) + return audit + + +def test_standalone_audit_metadata_hash_includes_referenced_jinja_macros(): + query = "SELECT * FROM db.tbl WHERE id <> {{ jone() }}" + unused = "{% macro unused() %}x{% endmacro %}" + + audit_1 = _load_standalone_audit_with_macros( + "{% macro jone() %}1{% endmacro %}" + unused, query + ) + audit_1_again = _load_standalone_audit_with_macros( + "{% macro jone() %}1{% endmacro %}" + unused, query + ) + audit_2 = _load_standalone_audit_with_macros( + "{% macro jone() %}2{% endmacro %}" + unused, query + ) + + assert audit_1.metadata_hash == audit_1_again.metadata_hash + assert audit_1.metadata_hash != audit_2.metadata_hash + + +def test_standalone_audit_metadata_hash_ignores_unreferenced_jinja_macros(): + query = "SELECT * FROM db.tbl WHERE id <> {{ jone() }}" + jone = "{% macro jone() %}1{% endmacro %}" + + audit_1 = _load_standalone_audit_with_macros( + jone + "{% macro unused() %}x{% endmacro %}", query + ) + audit_2 = _load_standalone_audit_with_macros( + jone + "{% macro unused() %}y{% endmacro %}", query + ) + + assert "unused" not in audit_1.jinja_macros.root_macros + assert audit_1.metadata_hash == audit_2.metadata_hash + + +def test_standalone_audit_metadata_hash_unchanged_without_jinja_macros(): + audit = load_audit( + parse( + """ + AUDIT ( + name my_audit, + owner owner_name, + standalone true, + ); + + SELECT * FROM db.tbl WHERE id IS NULL + """ + ) + ) + assert isinstance(audit, StandaloneAudit) + assert not audit.jinja_macros.root_macros and not audit.jinja_macros.packages + + # This literal must match the value computed on main before Jinja macros were hashed, so that + # audits without Jinja macros are not rehashed (and thus not re-versioned) on upgrade. + assert audit.metadata_hash == "3234547564" + + +def test_standalone_audit_metadata_hash_ignores_global_objs_without_macros(): + query = "SELECT * FROM db.tbl WHERE id IS NULL" + no_macros = "" + + empty = _load_standalone_audit_with_macros(no_macros, query) + with_globals = _load_standalone_audit_with_macros( + no_macros, query, global_objs={"refs": {"a": {"name": "a"}}, "vars": {"v": 1}} + ) + + assert with_globals.jinja_macros.global_objs + assert not with_globals.jinja_macros.root_macros and not with_globals.jinja_macros.packages + assert with_globals.metadata_hash == empty.metadata_hash + # Must match the value computed on main, see the test above. + assert empty.metadata_hash == "435209792" + + +def test_standalone_audit_metadata_hash_with_date_var(): + from datetime import date + + # dbt loads an unquoted YAML date as a `date` object, which `json.dumps` can't serialize. + audit = _load_standalone_audit_with_macros( + "{% macro jone() %}1{% endmacro %}", + "SELECT * FROM db.tbl WHERE id <> {{ jone() }}", + global_objs={"vars": {"cutoff": date(2024, 1, 2)}}, + ) + + assert audit.metadata_hash + + +def test_standalone_audit_metadata_hash_includes_referenced_package_macros(): + query = "SELECT * FROM db.tbl WHERE id <> {{ pkg.jone() }}" + + def load(body: str) -> StandaloneAudit: + return _load_standalone_audit_with_macros( + "", query, package_macros={"pkg": "{% macro jone() %}" + body + "{% endmacro %}"} + ) + + audit_1, audit_1_again, audit_2 = load("1"), load("1"), load("2") + + assert "pkg" in audit_1.jinja_macros.packages + assert not audit_1.jinja_macros.root_macros + assert audit_1.metadata_hash == audit_1_again.metadata_hash + assert audit_1.metadata_hash != audit_2.metadata_hash + + +def test_standalone_audit_metadata_hash_ignores_attribute_only_jinja(): + # `{{ target.name }}` reads an attribute and calls no macro, but trimming still leaves an empty + # `target` package in the registry. The hash must match the value computed on main before + # Jinja macros were hashed, so that such audits are not rehashed on upgrade. + audit = _load_standalone_audit_with_macros( + "", "SELECT * FROM db.tbl WHERE name = '{{ target.name }}'" + ) + assert audit.jinja_macros.packages == {"target": {}} + assert not audit.jinja_macros.root_macros + + assert audit.metadata_hash == "3822507146" diff --git a/tests/utils/test_jinja.py b/tests/utils/test_jinja.py index 01eeb47412..6106296f10 100644 --- a/tests/utils/test_jinja.py +++ b/tests/utils/test_jinja.py @@ -353,3 +353,29 @@ def test_builtin_b64decode_with_env_var(monkeypatch): config = yaml.load("password: \"{{ env_var('SNOWFLAKE_PW_B64') | b64decode }}\"") assert config == {"password": "super-secret-pw"} + + +def test_macro_registry_data_hash_values_with_non_json_vars(): + from datetime import date + + macros = MacroExtractor().extract("{% macro m() %}1{% endmacro %}") + + # dbt loads an unquoted YAML date such as `cutoff: 2024-01-02` as a `date` object. + registry = JinjaMacroRegistry(global_objs={"vars": {"cutoff": date(2024, 1, 2)}}) + registry.add_macros(macros) + assert registry.data_hash_values[-1] == '{"vars": {"cutoff": "2024-01-02"}}' + + # Saving a snapshot to state stores the date as a string. The hash must not change when the + # registry is loaded back, because migrations recompute fingerprints from stored snapshots. + reloaded = JinjaMacroRegistry.parse_raw(registry.json()) + assert reloaded.global_objs["vars"]["cutoff"] == "2024-01-02" + assert reloaded.data_hash_values == registry.data_hash_values + + # JSON-serializable globals produce the same values as before. + plain = JinjaMacroRegistry(global_objs={"vars": {"cutoff": "2024-01-02", "limit": 10}}) + plain.add_macros(macros) + assert plain.data_hash_values == [ + "m", + "{% macro m() %}1{% endmacro %}", + '{"vars": {"cutoff": "2024-01-02", "limit": 10}}', + ]