This is an automated email from the ASF dual-hosted git repository. chenjinbao1989 pushed a commit to branch cbdb-postgres-merge in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit dfb01c85a7560e429b937bda9ec17c617e09dc60 Author: Alexandra Wang <[email protected]> AuthorDate: Thu Apr 27 16:53:32 2023 -0700 Add gp_stat_progress_%_summary system views Added the following views: gp_stat_progress_vacuum_summary gp_stat_progress_analyze_summary gp_stat_progress_cluster_summary gp_stat_progress_create_index_summary Also replaced pg_stat_progress_* views with gp_stat_progress_* views for existing tests. --- src/backend/catalog/Makefile | 3 +- src/backend/catalog/system_views_gp_summary.sql | 109 ++++++++++++ src/backend/commands/analyze.c | 1 + src/bin/initdb/initdb.c | 6 +- src/test/isolation2/expected/analyze_progress.out | 95 +++++++++++ .../expected/ao_index_build_progress.out | 88 +++++++--- .../isolation2/expected/vacuum_progress_row.out | 189 +++++++++++++-------- src/test/isolation2/isolation2_schedule | 1 + src/test/isolation2/sql/analyze_progress.sql | 42 +++++ .../isolation2/sql/ao_index_build_progress.sql | 44 +++-- src/test/isolation2/sql/setup.sql | 21 +++ src/test/isolation2/sql/vacuum_progress_row.sql | 168 +++++------------- 12 files changed, 537 insertions(+), 230 deletions(-) diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index b45a3776e59..5679abf152a 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -204,6 +204,7 @@ endif $(INSTALL_DATA) $(srcdir)/system_functions.sql '$(DESTDIR)$(datadir)/system_functions.sql' $(INSTALL_DATA) $(srcdir)/system_views.sql '$(DESTDIR)$(datadir)/system_views.sql' $(INSTALL_DATA) $(srcdir)/$(GP_SYSVIEW_SQL) '$(DESTDIR)$(datadir)/$(GP_SYSVIEW_SQL)' + $(INSTALL_DATA) $(srcdir)/system_views_gp_summary.sql '$(DESTDIR)$(datadir)/system_views_gp_summary.sql' $(INSTALL_DATA) $(srcdir)/information_schema.sql '$(DESTDIR)$(datadir)/information_schema.sql' $(INSTALL_DATA) $(call vpathsearch,cdb_schema.sql) '$(DESTDIR)$(datadir)/cdb_init.d/cdb_schema.sql' $(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt' @@ -215,7 +216,7 @@ installdirs: .PHONY: uninstall-data uninstall-data: - rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt fix-CVE-2024-4317.sql) + rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql system_views_gp_summary.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt fix-CVE-2024-4317.sql) ifeq ($(USE_INTERNAL_FTS_FOUND), false) rm -f $(addprefix '$(DESTDIR)$(datadir)'/, external_fts.sql) endif diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql new file mode 100644 index 00000000000..58f4768c124 --- /dev/null +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -0,0 +1,109 @@ +/* + * Greenplum System Summary Views + * + * Portions Copyright (c) 2006-2010, Greenplum inc. + * Portions Copyright (c) 2012-Present VMware, Inc. or its affiliates. + * Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + * src/backend/catalog/system_views_gp_summary.sql + * + + * This file contains summary views for various Greenplum system catalog + * views. These summary views are designed to provide aggregated or averaged + * information for partitioned and replicated tables, considering multiple + * segments in a Greenplum database. + * + * Note: this file is read in single-user -j mode, which means that the + * command terminator is semicolon-newline-newline; whenever the backend + * sees that, it stops and executes what it's got. If you write a lot of + * statements without empty lines between, they'll all get quoted to you + * in any error message about one of them, so don't do that. Also, you + * cannot write a semicolon immediately followed by an empty line in a + * string literal (including a function body!) or a multiline comment. + */ + +CREATE VIEW gp_stat_progress_vacuum_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.phase, + case when d.policytype = 'r' then (sum(a.heap_blks_total)/d.numsegments)::bigint else sum(a.heap_blks_total) end heap_blks_total, + case when d.policytype = 'r' then (sum(a.heap_blks_scanned)/d.numsegments)::bigint else sum(a.heap_blks_scanned) end heap_blks_scanned, + case when d.policytype = 'r' then (sum(a.heap_blks_vacuumed)/d.numsegments)::bigint else sum(a.heap_blks_vacuumed) end heap_blks_vacuumed, + case when d.policytype = 'r' then (sum(a.index_vacuum_count)/d.numsegments)::bigint else sum(a.index_vacuum_count) end index_vacuum_count, + case when d.policytype = 'r' then (sum(a.max_dead_tuples)/d.numsegments)::bigint else sum(a.max_dead_tuples) end max_dead_tuples, + case when d.policytype = 'r' then (sum(a.num_dead_tuples)/d.numsegments)::bigint else sum(a.num_dead_tuples) end num_dead_tuples +FROM gp_stat_progress_vacuum a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_vacuum a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.phase, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_analyze_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.phase, + case when d.policytype = 'r' then (sum(a.sample_blks_total)/d.numsegments)::bigint else sum(a.sample_blks_total) end sample_blks_total, + case when d.policytype = 'r' then (sum(a.sample_blks_scanned)/d.numsegments)::bigint else sum(a.sample_blks_scanned) end sample_blks_scanned, + case when d.policytype = 'r' then (sum(a.ext_stats_total)/d.numsegments)::bigint else sum(a.ext_stats_total) end ext_stats_total, + case when d.policytype = 'r' then (sum(a.ext_stats_computed)/d.numsegments)::bigint else sum(a.ext_stats_computed) end ext_stats_computed, + case when d.policytype = 'r' then (sum(a.child_tables_total)/d.numsegments)::bigint else sum(a.child_tables_total) end child_tables_total, + case when d.policytype = 'r' then (sum(a.child_tables_done)/d.numsegments)::bigint else sum(a.child_tables_done) end child_tables_done +FROM gp_stat_progress_analyze a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_analyze a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.phase, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_cluster_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.command, + a.phase, + a.cluster_index_relid, + case when d.policytype = 'r' then (sum(a.heap_tuples_scanned)/d.numsegments)::bigint else sum(a.heap_tuples_scanned) end heap_tuples_scanned, + case when d.policytype = 'r' then (sum(a.heap_tuples_written)/d.numsegments)::bigint else sum(a.heap_tuples_written) end heap_tuples_written, + case when d.policytype = 'r' then (sum(a.heap_blks_total)/d.numsegments)::bigint else sum(a.heap_blks_total) end heap_blks_total, + case when d.policytype = 'r' then (sum(a.heap_blks_scanned)/d.numsegments)::bigint else sum(a.heap_blks_scanned) end heap_blks_scanned, + case when d.policytype = 'r' then (sum(a.index_rebuild_count)/d.numsegments)::bigint else sum(a.index_rebuild_count) end index_rebuild_count +FROM gp_stat_progress_cluster a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_cluster a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.command, a.phase, a.cluster_index_relid, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_create_index_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.index_relid, + a.command, + a.phase, + case when d.policytype = 'r' then (sum(a.lockers_total)/d.numsegments)::bigint else sum(a.lockers_total) end lockers_total, + case when d.policytype = 'r' then (sum(a.lockers_done)/d.numsegments)::bigint else sum(a.lockers_done) end lockers_done, + max(a.current_locker_pid) as current_locker_pid, + case when d.policytype = 'r' then (sum(a.blocks_total)/d.numsegments)::bigint else sum(a.blocks_total) end blocks_total, + case when d.policytype = 'r' then (sum(a.blocks_done)/d.numsegments)::bigint else sum(a.blocks_done) end blocks_done, + case when d.policytype = 'r' then (sum(a.tuples_total)/d.numsegments)::bigint else sum(a.tuples_total) end tuples_total, + case when d.policytype = 'r' then (sum(a.tuples_done)/d.numsegments)::bigint else sum(a.tuples_done) end tuples_done, + case when d.policytype = 'r' then (sum(a.partitions_total)/d.numsegments)::bigint else sum(a.partitions_total) end partitions_total, + case when d.policytype = 'r' then (sum(a.partitions_done)/d.numsegments)::bigint else sum(a.partitions_done) end partitions_done +FROM gp_stat_progress_create_index a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_create_index a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.index_relid, a.command, a.phase, d.policytype, d.numsegments; diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index f2fff2c8607..5f7b8faf8e1 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -1875,6 +1875,7 @@ acquire_sample_rows(Relation onerel, int elevel, pgstat_progress_update_param(PROGRESS_ANALYZE_BLOCKS_DONE, ++blksdone); + SIMPLE_FAULT_INJECTOR("analyze_block"); } ExecDropSingleTupleTableSlot(slot); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index ca0e5bbcd97..b58fc4cee90 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -193,6 +193,7 @@ static char *external_fts_files; static char *system_functions_file; static char *system_views_file; static char *system_views_gp_file; +static char *system_views_gp_summary_file; static bool success = false; static bool made_new_pgdata = false; static bool found_existing_pgdata = false; @@ -1767,8 +1768,6 @@ setup_run_file(FILE *cmdfd, const char *filename) } PG_CMD_PUTS("\n\n"); - - free(lines); } /* @@ -3037,6 +3036,7 @@ setup_data_file_paths(void) set_input(&system_functions_file, "system_functions.sql"); set_input(&system_views_file, "system_views.sql"); set_input(&system_views_gp_file, "system_views_gp.sql"); + set_input(&system_views_gp_summary_file, "system_views_gp_summary.sql"); set_input(&cdb_init_d_dir, "cdb_init.d"); @@ -3071,6 +3071,7 @@ setup_data_file_paths(void) check_input(system_functions_file); check_input(system_views_file); check_input(system_views_gp_file); + check_input(system_views_gp_summary_file); } @@ -3401,6 +3402,7 @@ initialize_data_directory(void) setup_run_file(cmdfd, system_views_file); setup_run_file(cmdfd, system_views_gp_file); + setup_run_file(cmdfd, system_views_gp_summary_file); setup_description(cmdfd); diff --git a/src/test/isolation2/expected/analyze_progress.out b/src/test/isolation2/expected/analyze_progress.out new file mode 100644 index 00000000000..97c02fbe617 --- /dev/null +++ b/src/test/isolation2/expected/analyze_progress.out @@ -0,0 +1,95 @@ +-- Test gp_stat_progress_analyze_summary +-- setup hash distributed table +CREATE TABLE t_analyze_part (a INT, b INT) DISTRIBUTED BY (a); +CREATE +INSERT INTO t_analyze_part SELECT i, i FROM generate_series(1, 100000) i; +INSERT 100000 + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- session 1: analyze the table +1&: ANALYZE t_analyze_part; <waiting ...> +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 60 blocks have been scanned as aggregated progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + has_pid | datname | relid | phase | sample_blks_total | sample_blks_scanned +---------+----------------+----------------+-----------------------+-------------------+--------------------- + t | isolation2test | t_analyze_part | acquiring sample rows | 111 | 60 +(1 row) + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> +ANALYZE + +-- teardown +DROP TABLE t_analyze_part; +DROP + +-- setup replicated table +CREATE TABLE t_analyze_repl (a INT, b INT) DISTRIBUTED REPLICATED; +CREATE +INSERT INTO t_analyze_repl SELECT i, i FROM generate_series(1, 100000) i; +INSERT 100000 + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- session 1: analyze the table +1&: ANALYZE t_analyze_repl; <waiting ...> +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 20 blocks have been scanned as average progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + has_pid | datname | relid | phase | sample_blks_total | sample_blks_scanned +---------+----------------+----------------+-----------------------+-------------------+--------------------- + t | isolation2test | t_analyze_repl | acquiring sample rows | 111 | 20 +(1 row) + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> +ANALYZE + +-- teardown +DROP TABLE t_analyze_repl; +DROP + diff --git a/src/test/isolation2/expected/ao_index_build_progress.out b/src/test/isolation2/expected/ao_index_build_progress.out index 1048076ce9f..2cc97d09031 100644 --- a/src/test/isolation2/expected/ao_index_build_progress.out +++ b/src/test/isolation2/expected/ao_index_build_progress.out @@ -8,36 +8,52 @@ CREATE -- Insert all tuples to seg1. INSERT INTO ao_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; INSERT 100000 +INSERT INTO ao_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT 100000 +INSERT INTO ao_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; +INSERT 100000 -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON ao_index_build_progress(i); <waiting ...> -- Wait until some AO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -1U: SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'ao_index_build_progress'::regclass; +SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'ao_index_build_progress'::regclass; command | phase | blocks_total_actual | blocks_total_reported | blocks_done_reported --------------+--------------------------------+---------------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 10 | 10 | 2 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'ao_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 30 | 6 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE @@ -49,38 +65,54 @@ CREATE -- Insert all tuples to seg1. INSERT INTO aoco_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; INSERT 100000 +INSERT INTO aoco_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT 100000 +INSERT INTO aoco_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; +INSERT 100000 -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON aoco_index_build_progress(i); <waiting ...> -- Wait until some AOCO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -- Note: all blocks for the relation have to be scanned as we are building an -- index for the first time and a block directory has to be created. -1U: SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'aoco_index_build_progress'::regclass; +SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; command | phase | blocks_total_actual | blocks_total_reported | blocks_done_reported --------------+--------------------------------+---------------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 20 | 20 | 4 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 60 | 12 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE @@ -88,36 +120,48 @@ CREATE -- Repeat the test for another index build -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON aoco_index_build_progress(j); <waiting ...> -- Wait until some AOCO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the size of col j's segfile. -- Note: since we already had a block directory prior to the index build on --- column 'j', only column 'j' will be scanned. CBDB_CHERRY_PICK_MERGE_FIXME: fix when ao blkdir will be supported -1U: SELECT command, phase, ((pg_stat_file(pg_relation_filepath('aoco_index_build_progress') || '.' || 129)).size + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'aoco_index_build_progress'::regclass; +-- column 'j', only column 'j' will be scanned. +1U: SELECT command, phase, ((pg_stat_file(pg_relation_filepath('aoco_index_build_progress') || '.' || 129)).size + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; command | phase | col_j_blocks | blocks_total_reported | blocks_done_reported --------------+--------------------------------+--------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 8 | 20 | 4 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 24 | 9 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out index 0f1b3e65ef5..619b3e41b80 100644 --- a/src/test/isolation2/expected/vacuum_progress_row.out +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -16,22 +16,21 @@ CREATE CREATE INDEX on vacuum_progress_ao_row(j); CREATE --- Insert all tuples to seg1 from two current sessions so that data are stored --- in two segment files. +-- Insert from two current sessions so that data are stored in two segment files. 1: BEGIN; BEGIN 2: BEGIN; BEGIN -1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -- Commit so that the logical EOF of segno 2 is non-zero. 2: COMMIT; COMMIT 2: BEGIN; BEGIN -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -- Abort so that segno 2 has dead tuples after its logical EOF 2: ABORT; @@ -53,6 +52,11 @@ SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_ (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | last_vacuum | vacuum_count +------------+------------+-------------+-------------- + 33327 | 66654 | | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | last_vacuum | vacuum_count ------------+------------+-------------+-------------- 100000 | 200000 | | 0 (1 row) @@ -60,11 +64,13 @@ SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables -- Perform VACUUM and observe the progress -- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1: set Debug_appendonly_print_compaction to on; SET @@ -75,85 +81,120 @@ SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2 Success: (1 row) -- We are in pre_cleanup phase and some blocks should've been vacuumed by now -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized pre-cleanup | 165 | 0 | 110 | 0 | 100000 | 0 + vacuum_progress_ao_row | append-optimized pre-cleanup | 55 | 0 | 37 | 0 | 33327 | 0 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 166 | 0 | 111 | 0 | 100000 | 0 (1 row) -- Resume execution and suspend again in the middle of compact phase -SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized compact | 165 | 0 | 110 | 0 | 100000 | 199 + vacuum_progress_ao_row | append-optimized compact | 55 | 0 | 37 | 0 | 33327 | 227 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 166 | 0 | 111 | 0 | 100000 | 594 (1 row) -- Resume execution and suspend again after compacting all segfiles -SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- After compacting all segfiles we expect 50000 dead tuples -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized compact | 165 | 55 | 110 | 0 | 100000 | 50000 + vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33327 | 16622 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 166 | 57 | 111 | 0 | 100000 | 50000 (1 row) -- Resume execution and entering post_cleaup phase, suspend at the end of it. -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized post-cleanup | 165 | 55 | 110 | 0 | 100000 | 50000 + vacuum_progress_ao_row | append-optimized post-cleanup | 55 | 19 | 37 | 0 | 33327 | 16622 (1 row) -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized post-cleanup | 166 | 57 | 111 | 0 | 100000 | 50000 +(1 row) + +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> VACUUM @@ -166,63 +207,68 @@ VACUUM SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; relpages | reltuples | relallvisible ----------+-----------+--------------- - 83 | 50000 | 0 + 84 | 50000 | 0 (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count ------------+------------+-----------------+-------------- - 50000 | 0 | t | 1 + 16705 | 0 | t | 1 (1 row) -- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_inject_fault ----------------- Success: (1 row) -1&: VACUUM vacuum_progress_ao_row; <waiting ...> --- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: +(2 rows) +1&: VACUUM vacuum_progress_ao_row; <waiting ...> +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes for segment 0. +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) --- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP for segment 1 and 2, there should be only 1. +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: +(2 rows) +-- Segment 0 is in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 0; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+-------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | vacuuming indexes | 83 | 0 | 0 | 1 | 50000 | 0 + vacuum_progress_ao_row | vacuuming indexes | 28 | 0 | 0 | 1 | 16737 | 0 (1 row) +-- Segment 1 and 2 are in truncate segments phase (part of ao post_cleanup phase), heap_blks_vacuumed should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > 0; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 28 | 0 | 19 | 2 | 16558 | 0 + vacuum_progress_ao_row | append-optimized pre-cleanup | 28 | 0 | 19 | 2 | 16705 | 0 +(2 rows) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 56 | 0 | 38 | 4 | 33263 | 0 + vacuum_progress_ao_row | vacuuming indexes | 28 | 0 | 0 | 1 | 16737 | 0 +(2 rows) --- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; - gp_inject_fault ------------------ Success: -(1 row) -SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; - gp_wait_until_triggered_fault -------------------------------- - Success: -(1 row) --- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples -------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized pre-cleanup | 83 | 0 | 55 | 2 | 50000 | 0 -(1 row) - -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: +(3 rows) +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_inject_fault ----------------- Success: @@ -230,25 +276,34 @@ SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) 1<: <... completed> VACUUM --- Vacuum has finished, nothing should show up in the progress view. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Vacuum has finished, nothing should show up in the view. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- +(0 rows) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- (0 rows) -- pg_class and collected stats view should be updated after the 2nd VACUUM -1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); - wait_until_dead_tup_change_to -------------------------------- - OK +1U: SELECT wait_until_vacuum_count_change_to('vacuum_progress_ao_row'::regclass::oid, 2); + wait_until_vacuum_count_change_to +----------------------------------- + OK (1 row) SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; relpages | reltuples | relallvisible ----------+-----------+--------------- - 28 | 50000 | 0 + 30 | 50000 | 0 (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 16705 | 0 | t | 2 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count ------------+------------+-----------------+-------------- 50000 | 0 | t | 2 (1 row) diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 18fd4a2c0c4..0b2a16544dc 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -234,6 +234,7 @@ test: idle_gang_cleaner # test idle_in_transaction_session_timeout test: ao_index_build_progress +test: analyze_progress # Tests for FTS test: fts_errors diff --git a/src/test/isolation2/sql/analyze_progress.sql b/src/test/isolation2/sql/analyze_progress.sql new file mode 100644 index 00000000000..311ab161eb2 --- /dev/null +++ b/src/test/isolation2/sql/analyze_progress.sql @@ -0,0 +1,42 @@ +-- Test gp_stat_progress_analyze_summary +-- setup hash distributed table +CREATE TABLE t_analyze_part (a INT, b INT) DISTRIBUTED BY (a); +INSERT INTO t_analyze_part SELECT i, i FROM generate_series(1, 100000) i; + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 1: analyze the table +1&: ANALYZE t_analyze_part; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 60 blocks have been scanned as aggregated progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +1<: + +-- teardown +DROP TABLE t_analyze_part; + +-- setup replicated table +CREATE TABLE t_analyze_repl (a INT, b INT) DISTRIBUTED REPLICATED; +INSERT INTO t_analyze_repl SELECT i, i FROM generate_series(1, 100000) i; + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 1: analyze the table +1&: ANALYZE t_analyze_repl; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 20 blocks have been scanned as average progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +1<: + +-- teardown +DROP TABLE t_analyze_repl; diff --git a/src/test/isolation2/sql/ao_index_build_progress.sql b/src/test/isolation2/sql/ao_index_build_progress.sql index 19aade99e29..c5e34d1eab7 100644 --- a/src/test/isolation2/sql/ao_index_build_progress.sql +++ b/src/test/isolation2/sql/ao_index_build_progress.sql @@ -7,30 +7,34 @@ CREATE TABLE ao_index_build_progress(i int, j bigint) USING ao_row -- Insert all tuples to seg1. INSERT INTO ao_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; +INSERT INTO ao_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT INTO ao_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON ao_index_build_progress(i); -- Wait until some AO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -1U: SELECT command, phase, +SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'ao_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'ao_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'ao_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: @@ -40,31 +44,35 @@ CREATE TABLE aoco_index_build_progress(i int, j int ENCODING (compresstype=zstd, -- Insert all tuples to seg1. INSERT INTO aoco_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; +INSERT INTO aoco_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT INTO aoco_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON aoco_index_build_progress(i); -- Wait until some AOCO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -- Note: all blocks for the relation have to be scanned as we are building an -- index for the first time and a block directory has to be created. -1U: SELECT command, phase, +SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'aoco_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: @@ -72,13 +80,13 @@ SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', d -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON aoco_index_build_progress(j); -- Wait until some AOCO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the size of col j's segfile. @@ -90,10 +98,12 @@ SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_succes AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'aoco_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: diff --git a/src/test/isolation2/sql/setup.sql b/src/test/isolation2/sql/setup.sql index 949f22f0002..6241413ba3b 100644 --- a/src/test/isolation2/sql/setup.sql +++ b/src/test/isolation2/sql/setup.sql @@ -461,6 +461,27 @@ begin end; /* in func */ $$ language plpgsql; +-- Helper function that ensures stats collector receives stat from the latest operation. +create or replace function wait_until_vacuum_count_change_to(relid oid, stat_val_expected bigint) + returns text as $$ +declare + stat_val int; /* in func */ + i int; /* in func */ +begin + i := 0; /* in func */ + while i < 1200 loop + select pg_stat_get_vacuum_count(relid) into stat_val; /* in func */ + if stat_val = stat_val_expected then /* in func */ + return 'OK'; /* in func */ + end if; /* in func */ + perform pg_sleep(0.1); /* in func */ + perform pg_stat_clear_snapshot(); /* in func */ + i := i + 1; /* in func */ + end loop; /* in func */ + return 'Fail'; /* in func */ +end; /* in func */ +$$ language plpgsql; + -- Helper function to get the number of blocks in a relation. CREATE OR REPLACE FUNCTION nblocks(rel regclass) RETURNS int AS $$ /* in func */ BEGIN /* in func */ diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql index 1ccf6ca090f..93da2a47bbc 100644 --- a/src/test/isolation2/sql/vacuum_progress_row.sql +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -11,16 +11,15 @@ CREATE TABLE vacuum_progress_ao_row(i int, j int); CREATE INDEX on vacuum_progress_ao_row(i); CREATE INDEX on vacuum_progress_ao_row(j); --- Insert all tuples to seg1 from two current sessions so that data are stored --- in two segment files. +-- Insert from two current sessions so that data are stored in two segment files. 1: BEGIN; 2: BEGIN; -1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; -- Commit so that the logical EOF of segno 2 is non-zero. 2: COMMIT; 2: BEGIN; -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; -- Abort so that segno 2 has dead tuples after its logical EOF 2: ABORT; 2q: @@ -32,40 +31,46 @@ DELETE FROM vacuum_progress_ao_row where j % 2 = 0; -- Lookup pg_class and collected stats view before VACUUM SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; -SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables WHERE relname = 'vacuum_progress_ao_row' and gp_segment_id = 1; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; -- Perform VACUUM and observe the progress -- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1: set Debug_appendonly_print_compaction to on; 1&: VACUUM vacuum_progress_ao_row; SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We are in pre_cleanup phase and some blocks should've been vacuumed by now -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and suspend again in the middle of compact phase -SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and suspend again after compacting all segfiles -SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- After compacting all segfiles we expect 50000 dead tuples -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and entering post_cleaup phase, suspend at the end of it. -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: -- pg_class and collected stats view should be updated after the 1st VACUUM @@ -74,113 +79,34 @@ SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_ SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; -- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; 1&: VACUUM vacuum_progress_ao_row; --- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; --- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - --- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; --- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes for segment 0. +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP for segment 1 and 2, there should be only 1. +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; +-- Segment 0 is in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 0; +-- Segment 1 and 2 are in truncate segments phase (part of ao post_cleanup phase), heap_blks_vacuumed should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > 0; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; 1<: --- Vacuum has finished, nothing should show up in the progress view. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Vacuum has finished, nothing should show up in the view. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- pg_class and collected stats view should be updated after the 2nd VACUUM -1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); +1U: SELECT wait_until_vacuum_count_change_to('vacuum_progress_ao_row'::regclass::oid, 2); SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; -SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables WHERE relname = 'vacuum_progress_ao_row' and gp_segment_id = 1; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; --- open if system views gp_stat_progress_vacuum* are enabled ---1q: ----- Test vacuum worker process is changed at post-cleanup phase due to mirror down. ----- Current behavior is it will clear previous compact phase num_dead_tuples in post-cleanup ----- phase (at injecting point vacuum_ao_post_cleanup_end), which is different from above case ----- in which vacuum worker isn't changed. ---ALTER SYSTEM SET gp_fts_mark_mirror_down_grace_period to 10; ---ALTER SYSTEM SET gp_fts_probe_interval to 10; ---SELECT gp_segment_id, pg_reload_conf() FROM gp_id UNION SELECT gp_segment_id, pg_reload_conf() FROM gp_dist_random('gp_id'); --- ---DROP TABLE IF EXISTS vacuum_progress_ao_row; ---CREATE TABLE vacuum_progress_ao_row(i int, j int); ---CREATE INDEX on vacuum_progress_ao_row(i); ---CREATE INDEX on vacuum_progress_ao_row(j); ---1: BEGIN; ---2: BEGIN; ---1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: COMMIT; ---2: BEGIN; ---2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: ABORT; ---2q: ---1: ABORT; ---DELETE FROM vacuum_progress_ao_row where j % 2 = 0; --- ----- Suspend execution at the end of compact phase. ---2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ---1: set debug_appendonly_print_compaction to on; ---1&: vacuum vacuum_progress_ao_row; --- ---2: SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 3, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ----- Non-zero progressing data num_dead_tuples is showed up. ---select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; ---select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- ----- Resume execution of compact phase and block at syncrep. ---2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; ---2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- stop the mirror should turn off syncrep ---2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; --- ----- Resume walsender to detect mirror down and suspend at the beginning ----- of post-cleanup taken over by a new vacuum worker. ---2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- resume walsender and let it exit so that mirror stop can be detected ---2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; ----- Ensure we enter into the target logic which stops cumulative data but ----- initializes a new vacrelstats at the beginning of post-cleanup phase. ----- Also all segments should reach to the same "vacuum_worker_changed" point ----- due to FTS version being changed. ---2: SELECT gp_wait_until_triggered_fault('vacuum_worker_changed', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- now seg1's mirror is marked as down ---2: SELECT content, role, preferred_role, mode, status FROM gp_segment_configuration WHERE content > -1; --- ----- Resume execution and entering post_cleaup phase, suspend at the end of it. ---2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ---2: SELECT gp_inject_fault('vacuum_worker_changed', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ---2: SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ----- The previous collected num_dead_tuples in compact phase is zero. ---select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; ---select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- ---2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ---1<: --- ----- restore environment ---1: reset debug_appendonly_print_compaction; --- ---2: SELECT pg_ctl_start(datadir, port) FROM gp_segment_configuration WHERE role = 'm' AND content = 1; ---2: SELECT wait_until_all_segments_synchronized(); --- ----- Cleanup ---SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; ---reset Debug_appendonly_print_compaction; ---reset default_table_access_method; ---ALTER SYSTEM RESET gp_fts_mark_mirror_down_grace_period; ---ALTER SYSTEM RESET gp_fts_probe_interval; ---SELECT gp_segment_id, pg_reload_conf() FROM gp_id UNION SELECT gp_segment_id, pg_reload_conf() FROM gp_dist_random('gp_id'); --- \ No newline at end of file +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; +reset Debug_appendonly_print_compaction; +reset default_table_access_method; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
