From f8b13e2514c895c61086d779ba809c97361c57bc Mon Sep 17 00:00:00 2001 From: talmacschen-arch Date: Sun, 1 Nov 2026 00:02:47 +0800 Subject: [PATCH] Back up Cloudberry incremental materialized views A Cloudberry incremental materialized view (IVM, pg_class.relisivm) was backed up as a plain `CREATE MATERIALIZED VIEW`, so on restore it lost its incremental behaviour. Worse, the IVM maintenance triggers the kernel installs on the base tables (named IVM_trigger_*, calling pg_catalog.ivm_immediate_before / ivm_immediate_maintenance) are not internal (tgisinternal = false), so GetTriggers backed them up too. Their names and function arguments embed the matview's OID, which changes on restore, and an incremental matview re-creates these triggers itself when it is repopulated -- so restoring the backed-up triggers would duplicate them with stale OIDs. This change: - selects relisivm in GetAllViews (Cloudberry only -- GPDB has no such column) and exposes it as View.IsIncremental; - emits `CREATE INCREMENTAL MATERIALIZED VIEW` for such views, so the kernel re-establishes the maintenance triggers when the matview is refreshed; - excludes the IVM maintenance triggers from GetTriggers (by trigger function, which is precise and a no-op on systems without IVM). Matview data handling is unchanged: matviews are still emitted `WITH NO DATA`; a manual `REFRESH MATERIALIZED VIEW` after restore repopulates the matview, at which point the kernel re-creates the correct maintenance triggers. Verified end to end on a live Apache Cloudberry 2.1.0 (SynxDB4 4.5.4-3) cluster with a rebuilt gpbackup/gprestore: - metadata.sql emits `CREATE INCREMENTAL MATERIALIZED VIEW` for the IVM and a plain `CREATE MATERIALIZED VIEW` for a normal matview, and contains none of the IVM_trigger_* statements; - after restore the IVM exists unpopulated (relisivm = t) with zero triggers; - after `REFRESH MATERIALIZED VIEW`, the kernel re-creates the 8 maintenance triggers and a subsequent INSERT into the base table is reflected in the matview (incremental maintenance works end to end). Add a unit test for the INCREMENTAL print path and integration tests for the relisivm flag and the IVM-trigger exclusion (both Cloudberry-gated). --- backup/predata_relations.go | 12 ++++++-- backup/predata_relations_other_test.go | 12 ++++++++ backup/queries_postdata.go | 15 +++++++++- backup/queries_relations.go | 11 ++++++- integration/postdata_queries_test.go | 17 +++++++++++ integration/predata_relations_queries_test.go | 29 +++++++++++++++++++ 6 files changed, 92 insertions(+), 4 deletions(-) diff --git a/backup/predata_relations.go b/backup/predata_relations.go index 4de2b542..3dfdd9f4 100644 --- a/backup/predata_relations.go +++ b/backup/predata_relations.go @@ -444,12 +444,20 @@ func PrintCreateViewStatement(metadataFile *utils.FileWithByteCount, objToc *toc if !view.IsMaterialized { metadataFile.MustPrintf("\n\nCREATE VIEW %s%s AS %s\n", view.FQN(), view.Options, view.Definition.String) } else { + // Cloudberry incremental materialized views (relisivm) must be recreated + // as INCREMENTAL so the kernel re-establishes their maintenance triggers; + // restoring them as a plain MATERIALIZED VIEW would silently drop the + // incremental behaviour. + matviewKeyword := "MATERIALIZED VIEW" + if view.IsIncremental { + matviewKeyword = "INCREMENTAL MATERIALIZED VIEW" + } accessMethodClause := "" if view.AccessMethodName != "" { accessMethodClause = fmt.Sprintf(" USING %s", view.AccessMethodName) } - metadataFile.MustPrintf("\n\nCREATE MATERIALIZED VIEW %s%s%s%s AS %s\nWITH NO DATA\n%s;\n", - view.FQN(), accessMethodClause, view.Options, tablespaceClause, view.Definition.String[:len(view.Definition.String)-1], view.DistPolicy.Policy) + metadataFile.MustPrintf("\n\nCREATE %s %s%s%s%s AS %s\nWITH NO DATA\n%s;\n", + matviewKeyword, view.FQN(), accessMethodClause, view.Options, tablespaceClause, view.Definition.String[:len(view.Definition.String)-1], view.DistPolicy.Policy) } section, entry := view.GetMetadataEntry() tier := globalTierMap[view.GetUniqueID()] diff --git a/backup/predata_relations_other_test.go b/backup/predata_relations_other_test.go index e52a7b91..984db5ad 100644 --- a/backup/predata_relations_other_test.go +++ b/backup/predata_relations_other_test.go @@ -660,6 +660,18 @@ GRANT ALL ON TABLE schema1.mview1 TO testrole;`} testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE MATERIALIZED VIEW schema1.mview1 WITH (security_barrier=true) TABLESPACE myTablespace AS SELECT count(*) FROM pg_tables WITH NO DATA +DISTRIBUTED BY (tablename);`) + }) + It("can print an incremental materialized view", func() { + // A Cloudberry incremental materialized view must be recreated with + // the INCREMENTAL keyword so the kernel re-establishes its maintenance + // triggers; otherwise it would be restored as a plain matview. + mview.IsIncremental = true + backup.PrintCreateViewStatement(backupfile, tocfile, mview, emptyMetadata) + testutils.ExpectEntry(tocfile.PredataEntries, 0, "schema1", "", "mview1", toc.OBJ_MATERIALIZED_VIEW) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, + `CREATE INCREMENTAL MATERIALIZED VIEW schema1.mview1 AS SELECT count(*) FROM pg_tables +WITH NO DATA DISTRIBUTED BY (tablename);`) }) }) diff --git a/backup/queries_postdata.go b/backup/queries_postdata.go index 1c73dd33..3294e7d5 100644 --- a/backup/queries_postdata.go +++ b/backup/queries_postdata.go @@ -439,6 +439,18 @@ func GetTriggers(connectionPool *dbconn.DBConn) []TriggerDefinition { if connectionPool.Version.IsGPDB() && connectionPool.Version.Before("6") { constraintClause = "tgisconstraint = 'f'" } + // Cloudberry incremental materialized views own maintenance triggers on their + // base tables that call pg_catalog.ivm_immediate_before / + // ivm_immediate_maintenance. They are re-created automatically when the + // incremental matview is repopulated (REFRESH) after restore, and their names + // and function arguments embed the matview's pre-restore OID, so backing them + // up would duplicate them with stale OIDs. Exclude them. The subquery is empty + // on systems without IVM, so this is a no-op there (not just Cloudberry). + ivmTriggerClause := `t.tgfoid NOT IN ( + SELECT p.oid FROM pg_proc p + JOIN pg_namespace pn ON pn.oid = p.pronamespace + WHERE pn.nspname = 'pg_catalog' + AND p.proname IN ('ivm_immediate_before', 'ivm_immediate_maintenance'))` query := fmt.Sprintf(` SELECT t.oid AS oid, quote_ident(t.tgname) AS name, @@ -452,8 +464,9 @@ func GetTriggers(connectionPool *dbconn.DBConn) []TriggerDefinition { AND tgname NOT LIKE 'pg_%%' AND %s AND %s + AND %s ORDER BY tgname`, - relationAndSchemaFilterClause(), constraintClause, ExtensionFilterClause("c")) + relationAndSchemaFilterClause(), constraintClause, ExtensionFilterClause("c"), ivmTriggerClause) results := make([]TriggerDefinition, 0) err := connectionPool.Select(&results, query) diff --git a/backup/queries_relations.go b/backup/queries_relations.go index 34652a84..bba9e51f 100644 --- a/backup/queries_relations.go +++ b/backup/queries_relations.go @@ -351,6 +351,7 @@ type View struct { Definition sql.NullString Tablespace string IsMaterialized bool + IsIncremental bool DistPolicy DistPolicy NeedsDummy bool ColumnDefs []ColumnDefinition @@ -429,6 +430,13 @@ func GetAllViews(connectionPool *dbconn.DBConn) []View { AND %s AND %s`, relationAndSchemaFilterClause(), ExtensionFilterClause("c")) + // relisivm (incremental materialized view flag) is a Cloudberry-only column; + // GPDB does not have it, so only reference it on Cloudberry. + incrementalExpr := "false" + if connectionPool.Version.IsCBDB() { + incrementalExpr = "c.relisivm" + } + atLeast7Query := fmt.Sprintf(` SELECT c.oid AS oid, @@ -438,6 +446,7 @@ func GetAllViews(connectionPool *dbconn.DBConn) []View { coalesce(' WITH (' || array_to_string(c.reloptions, ', ') || ')', '') AS options, coalesce(quote_ident(t.spcname), '') AS tablespace, c.relkind='m' AS ismaterialized, + %s AS isincremental, coalesce(quote_ident(am.amname), '') AS accessmethodname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace @@ -445,7 +454,7 @@ func GetAllViews(connectionPool *dbconn.DBConn) []View { LEFT JOIN pg_am am ON am.oid = c.relam WHERE c.relkind IN ('m', 'v') AND %s - AND %s`, relationAndSchemaFilterClause(), ExtensionFilterClause("c")) + AND %s`, incrementalExpr, relationAndSchemaFilterClause(), ExtensionFilterClause("c")) query := "" if connectionPool.Version.IsGPDB() && connectionPool.Version.Before("6") { diff --git a/integration/postdata_queries_test.go b/integration/postdata_queries_test.go index d44e1591..89f3b8e3 100644 --- a/integration/postdata_queries_test.go +++ b/integration/postdata_queries_test.go @@ -386,6 +386,23 @@ PARTITION BY RANGE (date) Expect(results).To(BeEmpty()) }) + It("does not include IVM maintenance triggers of an incremental materialized view", func() { + if !connectionPool.Version.IsCBDB() { + Skip("Incremental materialized views are only available on Cloudberry") + } + testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.ivm_base(id int, val int) DISTRIBUTED BY (id)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.ivm_base CASCADE") + // A populated incremental matview installs IVM_trigger_* maintenance + // triggers on the base table (calling pg_catalog.ivm_immediate_*). They + // are re-created automatically when the matview is repopulated after + // restore, so GetTriggers must not back them up. + testhelper.AssertQueryRuns(connectionPool, "CREATE INCREMENTAL MATERIALIZED VIEW public.ivm_mv AS SELECT id, val FROM public.ivm_base DISTRIBUTED BY (id)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP MATERIALIZED VIEW public.ivm_mv") + + results := backup.GetTriggers(connectionPool) + + Expect(results).To(BeEmpty()) + }) It("returns a slice of triggers for a specific schema", func() { triggerString1 := `CREATE TRIGGER sync_trigger_table1 AFTER INSERT OR DELETE OR UPDATE ON public.trigger_table1 FOR EACH STATEMENT EXECUTE PROCEDURE "RI_FKey_check_ins"()` triggerString2 := `CREATE TRIGGER sync_trigger_table1 AFTER INSERT OR DELETE OR UPDATE ON testschema.trigger_table1 FOR EACH STATEMENT EXECUTE PROCEDURE "RI_FKey_check_ins"()` diff --git a/integration/predata_relations_queries_test.go b/integration/predata_relations_queries_test.go index a0fc446e..a44192b4 100644 --- a/integration/predata_relations_queries_test.go +++ b/integration/predata_relations_queries_test.go @@ -553,5 +553,34 @@ PARTITION BY LIST (gender) Expect(results).To(HaveLen(1)) structmatcher.ExpectStructsToMatchExcluding(&materialView, &results[0], "Oid", "ColumnDefs", "DistPolicy.Oid") }) + It("sets IsIncremental for an incremental materialized view", func() { + // Cloudberry incremental materialized views (relisivm) must be flagged + // so they are recreated with CREATE INCREMENTAL MATERIALIZED VIEW; a + // plain materialized view must not be flagged. + if !connectionPool.Version.IsCBDB() { + Skip("Incremental materialized views are only available on Cloudberry") + } + testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.ivm_base(id int, val int) DISTRIBUTED BY (id)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.ivm_base CASCADE") + testhelper.AssertQueryRuns(connectionPool, "CREATE INCREMENTAL MATERIALIZED VIEW public.ivm_mv AS SELECT id, val FROM public.ivm_base DISTRIBUTED BY (id)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP MATERIALIZED VIEW public.ivm_mv") + testhelper.AssertQueryRuns(connectionPool, "CREATE MATERIALIZED VIEW public.plain_mv AS SELECT id, val FROM public.ivm_base DISTRIBUTED BY (id)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP MATERIALIZED VIEW public.plain_mv") + + results := backup.GetAllViews(connectionPool) + var ivm, plain backup.View + for _, v := range results { + switch v.Name { + case "ivm_mv": + ivm = v + case "plain_mv": + plain = v + } + } + Expect(ivm.IsMaterialized).To(BeTrue()) + Expect(ivm.IsIncremental).To(BeTrue()) + Expect(plain.IsMaterialized).To(BeTrue()) + Expect(plain.IsIncremental).To(BeFalse()) + }) }) })