diff --git a/contrib/pax_storage/src/test/regress/expected/gp_explain.out b/contrib/pax_storage/src/test/regress/expected/gp_explain.out index 6fc3b1ef844..91324daced5 100644 --- a/contrib/pax_storage/src/test/regress/expected/gp_explain.out +++ b/contrib/pax_storage/src/test/regress/expected/gp_explain.out @@ -41,6 +41,29 @@ begin end loop; end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; -- Test explain wal option CREATE TABLE explainwal (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. @@ -104,6 +127,23 @@ NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' a HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + plan_line +------------------------------------------------------------------ + Gather Motion N:N (slice1; segments: N) (actual rows=N loops=N) + -> Seq Scan on explaintest (actual rows=N loops=N) + Buffers: N + Planning: + Buffers: N + Optimizer: Postgres query optimizer +(6 rows) + EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- diff --git a/contrib/pax_storage/src/test/regress/expected/gp_explain_optimizer.out b/contrib/pax_storage/src/test/regress/expected/gp_explain_optimizer.out index 4244ab91437..06b25ac4b74 100644 --- a/contrib/pax_storage/src/test/regress/expected/gp_explain_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/gp_explain_optimizer.out @@ -41,6 +41,29 @@ begin end loop; end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; -- Test explain wal option CREATE TABLE explainwal (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. @@ -104,6 +127,23 @@ NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' a HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + plan_line +------------------------------------------------------------------ + Gather Motion N:N (slice1; segments: N) (actual rows=N loops=N) + -> Seq Scan on explaintest (actual rows=N loops=N) + Buffers: N + Planning: + Buffers: N + Optimizer: GPORCA +(6 rows) + EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN -------------------------------------------------------------------------------------------------------------------------- diff --git a/contrib/pax_storage/src/test/regress/sql/gp_explain.sql b/contrib/pax_storage/src/test/regress/sql/gp_explain.sql index 591d8f61540..2d873a2e908 100644 --- a/contrib/pax_storage/src/test/regress/sql/gp_explain.sql +++ b/contrib/pax_storage/src/test/regress/sql/gp_explain.sql @@ -45,6 +45,30 @@ begin end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; + -- Test explain wal option CREATE TABLE explainwal (id int4); @@ -94,6 +118,14 @@ CREATE TABLE explaintest (id int4); INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + EXPLAIN ANALYZE SELECT * FROM explaintest; set explain_memory_verbosity='summary'; diff --git a/src/backend/commands/explain_gp.c b/src/backend/commands/explain_gp.c index 5e9ff3376bd..b9016d816cf 100644 --- a/src/backend/commands/explain_gp.c +++ b/src/backend/commands/explain_gp.c @@ -818,6 +818,7 @@ cdbexplain_collectStatsFromNode(PlanState *planstate, CdbExplain_SendStatCtx *ct si->ntuples = instr->ntuples; si->ntuples2 = instr->ntuples2; si->nloops = instr->nloops; + si->bufusage = instr->bufusage; si->nfiltered1 = instr->nfiltered1; si->nfiltered2 = instr->nfiltered2; si->prf_work = instr->prf_work; @@ -953,7 +954,6 @@ cdbexplain_depStatAcc_saveText(CdbExplain_DepStatAcc *acc, } } /* cdbexplain_depStatAcc_saveText */ - /* * cdbexplain_depositStatsToNode * @@ -994,6 +994,7 @@ cdbexplain_depositStatsToNode(PlanState *planstate, CdbExplain_RecvStatCtx *ctx) CdbExplain_DepStatAcc vmem_reserved; CdbExplain_DepStatAcc totalPartTableScanned; CdbExplain_DepStatAcc sortSpaceUsed[NUM_SORT_SPACE_TYPE][NUM_SORT_METHOD]; + BufferUsage bufusage; int imsgptr; int nInst; @@ -1018,6 +1019,7 @@ cdbexplain_depositStatsToNode(PlanState *planstate, CdbExplain_RecvStatCtx *ctx) cdbexplain_depStatAcc_init0(&workmemwanted); cdbexplain_depStatAcc_init0(&totalWorkfileCreated); cdbexplain_depStatAcc_init0(&totalPartTableScanned); + memset(&bufusage, 0, sizeof(bufusage)); for (int i = 0; i < NUM_SORT_METHOD; i++) { for (int j = 0; j < NUM_SORT_SPACE_TYPE; j++) @@ -1063,6 +1065,7 @@ cdbexplain_depositStatsToNode(PlanState *planstate, CdbExplain_RecvStatCtx *ctx) cdbexplain_depStatAcc_upd(&workmemwanted, rsi->workmemwanted, rsh, rsi, nsi); cdbexplain_depStatAcc_upd(&totalWorkfileCreated, (rsi->workfileCreated ? 1 : 0), rsh, rsi, nsi); cdbexplain_depStatAcc_upd(&totalPartTableScanned, rsi->numPartScanned, rsh, rsi, nsi); + BufferUsageAdd(&bufusage, &rsi->bufusage); Assert(rsi->sortstats.sortMethod < NUM_SORT_METHOD); Assert(rsi->sortstats.spaceType < NUM_SORT_SPACE_TYPE); if (rsi->sortstats.sortMethod != SORT_TYPE_STILL_IN_PROGRESS) @@ -1134,6 +1137,9 @@ cdbexplain_depositStatsToNode(PlanState *planstate, CdbExplain_RecvStatCtx *ctx) instr->firststart = nodeAcc->nsimax->firststart; } + /* Buffer usage is summed across all workers, not taken from the winner. */ + instr->bufusage = bufusage; + /* Save extra message text for the most interesting winning qExecs. */ if (ctx->extratextbuf) { diff --git a/src/backend/executor/instrument.c b/src/backend/executor/instrument.c index 3fd39dfb167..354a029c440 100644 --- a/src/backend/executor/instrument.c +++ b/src/backend/executor/instrument.c @@ -32,7 +32,6 @@ static BufferUsage save_pgBufferUsage; WalUsage pgWalUsage; static WalUsage save_pgWalUsage; -static void BufferUsageAdd(BufferUsage *dst, const BufferUsage *add); static void WalUsageAdd(WalUsage *dst, WalUsage *add); /* GPDB specific */ @@ -87,6 +86,7 @@ InstrInit(Instrumentation *instr, int instrument_options) instr->need_bufusage = (instrument_options & INSTRUMENT_BUFFERS) != 0; instr->need_walusage = (instrument_options & INSTRUMENT_WAL) != 0; instr->need_timer = (instrument_options & INSTRUMENT_TIMER) != 0; + instr->need_cdb = (instrument_options & INSTRUMENT_CDB) != 0; } /* Entry to a plan node */ @@ -271,7 +271,7 @@ InstrAccumParallelQuery(BufferUsage *bufusage, WalUsage *walusage) } /* dst += add */ -static void +void BufferUsageAdd(BufferUsage *dst, const BufferUsage *add) { dst->shared_blks_hit += add->shared_blks_hit; @@ -506,8 +506,9 @@ pickInstrFromShmem(const Plan *plan, int instrument_options) MemoryContextSwitchTo(contextSave); } - if (NULL != instr && instrument_options & (INSTRUMENT_TIMER | INSTRUMENT_CDB)) + if (instr != NULL) { + instr->need_bufusage = (instrument_options & INSTRUMENT_BUFFERS) != 0; instr->need_timer = (instrument_options & INSTRUMENT_TIMER) != 0; instr->need_cdb = (instrument_options & INSTRUMENT_CDB) != 0; } diff --git a/src/include/commands/explain_gp.h b/src/include/commands/explain_gp.h index 016e5449543..c9210507e9f 100644 --- a/src/include/commands/explain_gp.h +++ b/src/include/commands/explain_gp.h @@ -34,6 +34,7 @@ typedef struct CdbExplain_StatInst double ntuples; /* Total tuples produced */ double ntuples2; double nloops; /* # of run cycles for this node */ + BufferUsage bufusage; /* Total buffer usage */ double nfiltered1; double nfiltered2; bool prf_work; diff --git a/src/include/executor/instrument.h b/src/include/executor/instrument.h index bce4b33c2f8..1fba5b70de3 100644 --- a/src/include/executor/instrument.h +++ b/src/include/executor/instrument.h @@ -64,7 +64,7 @@ typedef enum InstrumentOption { INSTRUMENT_NONE = 0, INSTRUMENT_TIMER = 1 << 0, /* needs timer (and row counts) */ - INSTRUMENT_BUFFERS = 1 << 1, /* needs buffer usage (not implemented yet) */ + INSTRUMENT_BUFFERS = 1 << 1, /* needs buffer usage */ INSTRUMENT_ROWS = 1 << 2, /* needs row count */ INSTRUMENT_WAL = 1 << 3, /* needs WAL usage */ INSTRUMENT_MEMORY_DETAIL = 0x20000000, /* needs detailed memory accounting */ @@ -135,6 +135,7 @@ extern void InstrAggNode(Instrumentation *dst, Instrumentation *add); extern void InstrStartParallelQuery(void); extern void InstrEndParallelQuery(BufferUsage *bufusage, WalUsage *walusage); extern void InstrAccumParallelQuery(BufferUsage *bufusage, WalUsage *walusage); +extern void BufferUsageAdd(BufferUsage *dst, const BufferUsage *add); extern void BufferUsageAccumDiff(BufferUsage *dst, const BufferUsage *add, const BufferUsage *sub); extern void WalUsageAccumDiff(WalUsage *dst, const WalUsage *add, diff --git a/src/test/regress/expected/gp_explain.out b/src/test/regress/expected/gp_explain.out index 6fc3b1ef844..91324daced5 100644 --- a/src/test/regress/expected/gp_explain.out +++ b/src/test/regress/expected/gp_explain.out @@ -41,6 +41,29 @@ begin end loop; end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; -- Test explain wal option CREATE TABLE explainwal (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. @@ -104,6 +127,23 @@ NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' a HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + plan_line +------------------------------------------------------------------ + Gather Motion N:N (slice1; segments: N) (actual rows=N loops=N) + -> Seq Scan on explaintest (actual rows=N loops=N) + Buffers: N + Planning: + Buffers: N + Optimizer: Postgres query optimizer +(6 rows) + EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/expected/gp_explain_optimizer.out b/src/test/regress/expected/gp_explain_optimizer.out index 4244ab91437..06b25ac4b74 100644 --- a/src/test/regress/expected/gp_explain_optimizer.out +++ b/src/test/regress/expected/gp_explain_optimizer.out @@ -41,6 +41,29 @@ begin end loop; end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; -- Test explain wal option CREATE TABLE explainwal (id int4); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. @@ -104,6 +127,23 @@ NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' a HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + plan_line +------------------------------------------------------------------ + Gather Motion N:N (slice1; segments: N) (actual rows=N loops=N) + -> Seq Scan on explaintest (actual rows=N loops=N) + Buffers: N + Planning: + Buffers: N + Optimizer: GPORCA +(6 rows) + EXPLAIN ANALYZE SELECT * FROM explaintest; QUERY PLAN -------------------------------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/sql/gp_explain.sql b/src/test/regress/sql/gp_explain.sql index 2b5aaa0c722..448fd0f8701 100644 --- a/src/test/regress/sql/gp_explain.sql +++ b/src/test/regress/sql/gp_explain.sql @@ -45,6 +45,30 @@ begin end; $$ language plpgsql; +-- Run EXPLAIN ANALYZE BUFFERS retaining the position of every Buffers: line +create or replace function explain_buffers_filter(explain_query text) returns setof text as +$$ +declare + explainrow text; +begin + for explainrow in execute + 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, SUMMARY OFF) ' || explain_query + loop + if explainrow ~ '^\s*Buffers:' then + explainrow := regexp_replace(explainrow, 'Buffers:.*$', 'Buffers: N'); + else + explainrow := regexp_replace(explainrow, + '\m\d+([kK]B?)\M', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '\m\d+(x)', 'N\1', 'g'); + explainrow := regexp_replace(explainrow, + '-?\m\d+\M', 'N', 'g'); + end if; + return next explainrow; + end loop; +end; +$$ language plpgsql; + -- Test explain wal option CREATE TABLE explainwal (id int4); @@ -94,6 +118,14 @@ CREATE TABLE explaintest (id int4); INSERT INTO explaintest SELECT generate_series(1, 10); ANALYZE explaintest; +-- Compare normalized buffer placement without automatic EXPLAIN processing. +-- explain_processing_off +SELECT plan_line +FROM explain_buffers_filter($$ + SELECT * FROM explaintest; +$$) WITH ORDINALITY AS plan(plan_line, line_number) +ORDER BY line_number; + EXPLAIN ANALYZE SELECT * FROM explaintest; set explain_memory_verbosity='summary';