On Fri, Aug 14, 2020 at 09:29:45AM +0900, Michael Paquier wrote: > Once this gets done, we should then be able to get rid of the extra > session locking taken when building the list of partitions, limiting > session locks to only be taken during the concurrent reindex of a > single partition (the table itself for a partition table, and the > parent table for a partition index), making the whole operation less > invasive.
The problem with dropped relations in REINDEX has been addressed by 1d65416, so I have gone through this patch again and simplified the use of session locks, these being taken only when doing a REINDEX CONCURRENTLY for a given partition. This part is in a rather committable shape IMO, so I would like to get it done first, before looking more at the other cases with CIC and CLUSTER. I am still planning to go through it once again. -- Michael
From 618760af9ce3229f775e1fdfb635837865aadb66 Mon Sep 17 00:00:00 2001 From: Michael Paquier <mich...@paquier.xyz> Date: Wed, 2 Sep 2020 10:37:46 +0900 Subject: [PATCH] Add support for partitioned tables and indexes with REINDEX Until now, REINDEX was not able to work with partitioned tables and indexes, forcing users to reindex partitions one by one. This extends REINDEX INDEX and REINDEX TABLE so as they can accept a partitioned index and table in input, respectively, to reindex all the partitions assigned to them. This shares some logic with schema and database REINDEX as each partition gets processed in its own transaction (this choice minimizes the number of invalid indexes to one partition with REINDEX CONCURRENTLY in case of a cancellation or failure in-flight, and the duration of each transaction). Isolation tests are added to emulate cases I bumped into, particularly with the concurrent drop of a leaf partition reindexed. Per their multi-transaction nature, these new flavors cannot run in a transaction block. Author: Justin Pryzby, Michael Paquier Discussion: https://postgr.es/m/db12e897-73ff-467e-94cb-4af03705435f.adger...@alibaba-inc.com --- src/include/commands/defrem.h | 6 +- src/backend/catalog/index.c | 25 +- src/backend/commands/indexcmds.c | 242 ++++++++++++++---- src/backend/tcop/utility.c | 6 +- .../isolation/expected/reindex-partitions.out | 107 ++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/reindex-partitions.spec | 61 +++++ src/test/regress/expected/create_index.out | 158 +++++++++++- src/test/regress/sql/create_index.sql | 88 ++++++- doc/src/sgml/ref/reindex.sgml | 13 +- .../postgres_fdw/expected/postgres_fdw.out | 21 ++ contrib/postgres_fdw/sql/postgres_fdw.sql | 20 ++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 657 insertions(+), 92 deletions(-) create mode 100644 src/test/isolation/expected/reindex-partitions.out create mode 100644 src/test/isolation/specs/reindex-partitions.spec diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index c26a102b17..22db3f660d 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -34,8 +34,10 @@ extern ObjectAddress DefineIndex(Oid relationId, bool check_not_in_use, bool skip_build, bool quiet); -extern void ReindexIndex(RangeVar *indexRelation, int options, bool concurrent); -extern Oid ReindexTable(RangeVar *relation, int options, bool concurrent); +extern void ReindexIndex(RangeVar *indexRelation, int options, bool concurrent, + bool isTopLevel); +extern Oid ReindexTable(RangeVar *relation, int options, bool concurrent, + bool isTopLevel); extern void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, int options, bool concurrent); extern char *makeObjectName(const char *name1, const char *name2, diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1d662e9af4..1764739ff5 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -77,6 +77,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" +#include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/tuplesort.h" @@ -3473,11 +3474,12 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, iRel->rd_rel->relam); /* - * The case of reindexing partitioned tables and indexes is handled - * differently by upper layers, so this case shouldn't arise. + * Partitioned indexes should never get processed here, as they have no + * physical storage. */ if (iRel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) - elog(ERROR, "unsupported relation kind for index \"%s\"", + elog(ERROR, "cannot reindex partitioned index \"%s.%s\"", + get_namespace_name(RelationGetNamespace(iRel)), RelationGetRelationName(iRel)); /* @@ -3694,20 +3696,13 @@ reindex_relation(Oid relid, int flags, int options) return false; /* - * This may be useful when implemented someday; but that day is not today. - * For now, avoid erroring out when called in a multi-table context - * (REINDEX SCHEMA) and happen to come across a partitioned table. The - * partitions may be reindexed on their own anyway. + * Partitioned tables should never get processed here, as they have no + * physical storage. */ if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - { - ereport(WARNING, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX of partitioned tables is not yet implemented, skipping \"%s\"", - RelationGetRelationName(rel)))); - table_close(rel, ShareLock); - return false; - } + elog(ERROR, "cannot reindex partitioned table \"%s.%s\"", + get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel)); toast_relid = rel->rd_rel->reltoastrelid; diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b3a92381f9..75008eebde 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -88,7 +88,12 @@ static List *ChooseIndexColumnNames(List *indexElems); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg); static bool ReindexRelationConcurrently(Oid relationOid, int options); -static void ReindexPartitionedIndex(Relation parentIdx); + +static void reindex_partitions(Oid relid, int options, bool concurrent, + bool isTopLevel); +static void reindex_multiple_internal(List *relids, int options, + bool concurrent); +static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -101,6 +106,16 @@ struct ReindexIndexCallbackState Oid locked_table_oid; /* tracks previously locked table */ }; +/* + * callback arguments for reindex_error_callback() + */ +typedef struct ReindexErrorInfo +{ + char *relname; + char *relnamespace; + char relkind; +} ReindexErrorInfo; + /* * CheckIndexCompatible * Determine whether an existing index definition is compatible with a @@ -2420,11 +2435,10 @@ ChooseIndexColumnNames(List *indexElems) * Recreate a specific index. */ void -ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) +ReindexIndex(RangeVar *indexRelation, int options, bool concurrent, bool isTopLevel) { struct ReindexIndexCallbackState state; Oid indOid; - Relation irel; char persistence; /* @@ -2445,22 +2459,10 @@ ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) RangeVarCallbackForReindexIndex, &state); - /* - * Obtain the current persistence of the existing index. We already hold - * lock on the index. - */ - irel = index_open(indOid, NoLock); - - if (irel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) - { - ReindexPartitionedIndex(irel); - return; - } - - persistence = irel->rd_rel->relpersistence; - index_close(irel, NoLock); - - if (concurrent && persistence != RELPERSISTENCE_TEMP) + persistence = get_rel_persistence(indOid); + if (get_rel_relkind(indOid) == RELKIND_PARTITIONED_INDEX) + reindex_partitions(indOid, options, concurrent, isTopLevel); + else if (concurrent && persistence != RELPERSISTENCE_TEMP) ReindexRelationConcurrently(indOid, options); else reindex_index(indOid, false, persistence, @@ -2542,7 +2544,7 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * Recreate all indexes of a table (and of its toast table, if any) */ Oid -ReindexTable(RangeVar *relation, int options, bool concurrent) +ReindexTable(RangeVar *relation, int options, bool concurrent, bool isTopLevel) { Oid heapOid; bool result; @@ -2560,7 +2562,9 @@ ReindexTable(RangeVar *relation, int options, bool concurrent) 0, RangeVarCallbackOwnsTable, NULL); - if (concurrent && get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) + if (get_rel_relkind(heapOid) == RELKIND_PARTITIONED_TABLE) + reindex_partitions(heapOid, options, concurrent, isTopLevel); + else if (concurrent && get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) { result = ReindexRelationConcurrently(heapOid, options); @@ -2604,7 +2608,6 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, MemoryContext private_context; MemoryContext old; List *relids = NIL; - ListCell *l; int num_keys; bool concurrent_warning = false; @@ -2688,11 +2691,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Only regular tables and matviews can have indexes, so ignore any * other kind of relation. * - * It is tempting to also consider partitioned tables here, but that - * has the problem that if the children are in the same schema, they - * would be processed twice. Maybe we could have a separate list of - * partitioned tables, and expand that afterwards into relids, - * ignoring any duplicates. + * Partitioned tables/indexes are skipped but matching leaf partitions + * are processed. */ if (classtuple->relkind != RELKIND_RELATION && classtuple->relkind != RELKIND_MATVIEW) @@ -2755,14 +2755,156 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, table_endscan(scan); table_close(relationRelation, AccessShareLock); - /* Now reindex each rel in a separate transaction */ + /* + * Process each relation listed in a separate transaction. Note that this + * commits and then starts a new transaction immediately. + */ + reindex_multiple_internal(relids, options, concurrent); + + MemoryContextDelete(private_context); +} + +/* + * Error callback specific to reindex_partitions(). + */ +static void +reindex_error_callback(void *arg) +{ + ReindexErrorInfo *errinfo = (ReindexErrorInfo *) arg; + + Assert(errinfo->relkind == RELKIND_PARTITIONED_INDEX || + errinfo->relkind == RELKIND_PARTITIONED_TABLE); + + if (errinfo->relkind == RELKIND_PARTITIONED_TABLE) + errcontext("while reindexing partitioned table \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); + else if (errinfo->relkind == RELKIND_PARTITIONED_INDEX) + errcontext("while reindexing partitioned index \"%s.%s\"", + errinfo->relnamespace, errinfo->relname); +} + +/* + * reindex_partitions + * + * Reindex a set of partitions, per the partitioned index or table given + * by the caller. + */ +static void +reindex_partitions(Oid relid, int options, bool concurrent, + bool isTopLevel) +{ + List *partitions = NIL; + char relkind = get_rel_relkind(relid); + char *relname = get_rel_name(relid); + char *relnamespace = get_namespace_name(get_rel_namespace(relid)); + MemoryContext reindex_context; + List *inhoids; + ListCell *lc; + ErrorContextCallback errcallback; + ReindexErrorInfo errinfo; + + Assert(relkind == RELKIND_PARTITIONED_INDEX || + relkind == RELKIND_PARTITIONED_TABLE); + + /* + * Check if this runs in a transaction block, with an error callback to + * provide the context under which a problem happens. + */ + errinfo.relname = pstrdup(relname); + errinfo.relnamespace = pstrdup(relnamespace); + errinfo.relkind = relkind; + errcallback.callback = reindex_error_callback; + errcallback.arg = (void *) &errinfo; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + PreventInTransactionBlock(isTopLevel, + relkind == RELKIND_PARTITIONED_TABLE ? + "REINDEX TABLE" : "REINDEX INDEX"); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Create special memory context for cross-transaction storage. + * + * Since it is a child of PortalContext, it will go away eventually even + * if we suffer an error so there is no need for special abort cleanup + * logic. + */ + reindex_context = AllocSetContextCreate(PortalContext, "Reindex", + ALLOCSET_DEFAULT_SIZES); + + /* ShareLock is enough to prevent schema modifications */ + inhoids = find_all_inheritors(relid, ShareLock, NULL); + + /* + * The list of relations to reindex are the physical partitions of the + * tree so discard any partitioned table or index. + */ + foreach(lc, inhoids) + { + Oid partoid = lfirst_oid(lc); + Oid parentoid; + char partkind = get_rel_relkind(partoid); + MemoryContext old_context; + + /* + * This discards partitioned tables, partitioned indexes and foreign + * tables. + */ + if (!RELKIND_HAS_STORAGE(partkind)) + continue; + + Assert(partkind == RELKIND_INDEX || + partkind == RELKIND_RELATION); + + parentoid = (partkind == RELKIND_INDEX) ? + IndexGetRelation(partoid, false) : partoid; + + /* Save partition OID */ + old_context = MemoryContextSwitchTo(reindex_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(old_context); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + reindex_multiple_internal(partitions, options, concurrent); + + /* + * Clean up working storage --- note we must do this after + * StartTransactionCommand, else we might be trying to delete the active + * context! + */ + MemoryContextDelete(reindex_context); +} + +/* + * reindex_multiple_internal + * + * Reindex a list of relations, each one being processed in its own + * transaction. This commits the existing transaction immediately, + * and starts a new transaction when finished. + */ +static void +reindex_multiple_internal(List *relids, int options, bool concurrent) +{ + ListCell *l; + PopActiveSnapshot(); CommitTransactionCommand(); + foreach(l, relids) { Oid relid = lfirst_oid(l); + char relkind; + char relpersistence; StartTransactionCommand(); + /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); @@ -2774,6 +2916,16 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, continue; } + relkind = get_rel_relkind(relid); + relpersistence = get_rel_persistence(relid); + + /* + * Partitioned tables and indexes can never be processed directly, and + * a list of their leaves should be built first. + */ + Assert(relkind != RELKIND_PARTITIONED_INDEX && + relkind != RELKIND_PARTITIONED_TABLE); + if (concurrent && get_rel_persistence(relid) != RELPERSISTENCE_TEMP) { (void) ReindexRelationConcurrently(relid, @@ -2781,6 +2933,12 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, REINDEXOPT_MISSING_OK); /* ReindexRelationConcurrently() does the verbose output */ } + else if (relkind == RELKIND_INDEX) + { + reindex_index(relid, false, relpersistence, + options | REINDEXOPT_REPORT_PROGRESS); + PopActiveSnapshot(); + } else { bool result; @@ -2803,9 +2961,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, CommitTransactionCommand(); } - StartTransactionCommand(); - MemoryContextDelete(private_context); + StartTransactionCommand(); } @@ -2817,8 +2974,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * view. For tables and materialized views, all its indexes will be rebuilt, * excluding invalid indexes and any indexes used in exclusion constraints, * but including its associated toast table indexes. For indexes, the index - * itself will be rebuilt. If 'relationOid' belongs to a partitioned table - * then we issue a warning to mention these are not yet supported. + * itself will be rebuilt. * * The locks taken on parent tables and involved indexes are kept until the * transaction is committed, at which point a session lock is taken on each @@ -3057,13 +3213,9 @@ ReindexRelationConcurrently(Oid relationOid, int options) MemoryContextSwitchTo(oldcontext); break; } + case RELKIND_PARTITIONED_TABLE: - /* see reindex_relation() */ - ereport(WARNING, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX of partitioned tables is not yet implemented, skipping \"%s\"", - get_rel_name(relationOid)))); - return false; + case RELKIND_PARTITIONED_INDEX: default: /* Return error if type of relation is not supported */ ereport(ERROR, @@ -3530,20 +3682,6 @@ ReindexRelationConcurrently(Oid relationOid, int options) return true; } -/* - * ReindexPartitionedIndex - * Reindex each child of the given partitioned index. - * - * Not yet implemented. - */ -static void -ReindexPartitionedIndex(Relation parentIdx) -{ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX is not yet implemented for partitioned indexes"))); -} - /* * Insert or delete an appropriate pg_inherits tuple to make the given index * be a partition of the indicated parent index. diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 6154d2c8c6..35553e04ce 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -926,10 +926,12 @@ standard_ProcessUtility(PlannedStmt *pstmt, switch (stmt->kind) { case REINDEX_OBJECT_INDEX: - ReindexIndex(stmt->relation, stmt->options, stmt->concurrent); + ReindexIndex(stmt->relation, stmt->options, + stmt->concurrent, isTopLevel); break; case REINDEX_OBJECT_TABLE: - ReindexTable(stmt->relation, stmt->options, stmt->concurrent); + ReindexTable(stmt->relation, stmt->options, + stmt->concurrent, isTopLevel); break; case REINDEX_OBJECT_SCHEMA: case REINDEX_OBJECT_SYSTEM: diff --git a/src/test/isolation/expected/reindex-partitions.out b/src/test/isolation/expected/reindex-partitions.out new file mode 100644 index 0000000000..5373abde25 --- /dev/null +++ b/src/test/isolation/expected/reindex-partitions.out @@ -0,0 +1,107 @@ +Parsed test spec with 3 sessions + +starting permutation: begin1 lockexcl1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockexcl1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockexcl1: LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockshare1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockshare1: LOCK reind_conc_parent IN SHARE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step drop3: <... completed> + +starting permutation: begin1 lockshare1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockshare1: LOCK reind_conc_parent IN SHARE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockupdate1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockupdate1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockupdate1: LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartexcl1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartexcl1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartexcl1: LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartshare1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step drop3: <... completed> + +starting permutation: begin1 lockpartshare1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartshare1: LOCK reind_conc_10_20 IN SHARE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartupdate1 reindex2 drop3 end1 +step begin1: BEGIN; +step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; +step reindex2: REINDEX TABLE reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex2: <... completed> +step drop3: <... completed> + +starting permutation: begin1 lockpartupdate1 reindex_conc2 drop3 end1 +step begin1: BEGIN; +step lockpartupdate1: LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; +step reindex_conc2: REINDEX TABLE CONCURRENTLY reind_conc_parent; <waiting ...> +step drop3: DROP TABLE reind_conc_10_20; <waiting ...> +step end1: COMMIT; +step reindex_conc2: <... completed> +step drop3: <... completed> diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 6acbb695ec..65d1443ac6 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -50,6 +50,7 @@ test: lock-committed-keyupdate test: update-locked-tuple test: reindex-concurrently test: reindex-schema +test: reindex-partitions test: propagate-lock-delete test: tuplelock-conflict test: tuplelock-update diff --git a/src/test/isolation/specs/reindex-partitions.spec b/src/test/isolation/specs/reindex-partitions.spec new file mode 100644 index 0000000000..738473648d --- /dev/null +++ b/src/test/isolation/specs/reindex-partitions.spec @@ -0,0 +1,61 @@ +# REINDEX with partitioned tables +# +# Ensure that concurrent and non-concurrent operations work correctly when +# a REINDEX is performed on a partitioned table or index. + +# Note that this does not test for partition creation in parallel of a +# REINDEX. The initial transaction of REINDEX building the partition list +# would make the CREATE TABLE command to wait, and once the first +# transaction commits it could be possible for the CREATE TABLE command +# to finish before the concurrent REINDEX is done, making the output +# unpredictable. + +setup +{ + CREATE TABLE reind_conc_parent (id int) PARTITION BY RANGE (id); + CREATE TABLE reind_conc_0_10 PARTITION OF reind_conc_parent + FOR VALUES FROM (0) TO (10); + CREATE TABLE reind_conc_10_20 PARTITION OF reind_conc_parent + FOR VALUES FROM (10) TO (20); + INSERT INTO reind_conc_parent VALUES (generate_series(0, 19)); +} + +teardown +{ + DROP TABLE reind_conc_parent; +} + +session "s1" +step "begin1" { BEGIN; } +step "lockexcl1" { LOCK reind_conc_parent IN ACCESS EXCLUSIVE MODE; } +step "lockshare1" { LOCK reind_conc_parent IN SHARE MODE; } +step "lockupdate1" { LOCK reind_conc_parent IN SHARE UPDATE EXCLUSIVE MODE; } +step "lockpartexcl1" { LOCK reind_conc_10_20 IN ACCESS EXCLUSIVE MODE; } +step "lockpartshare1" { LOCK reind_conc_10_20 IN SHARE MODE; } +step "lockpartupdate1" { LOCK reind_conc_10_20 IN SHARE UPDATE EXCLUSIVE MODE; } +step "end1" { COMMIT; } + +session "s2" +step "reindex2" { REINDEX TABLE reind_conc_parent; } +step "reindex_conc2" { REINDEX TABLE CONCURRENTLY reind_conc_parent; } + +session "s3" +step "drop3" { DROP TABLE reind_conc_10_20; } + +# An existing partition leaf is dropped after reindex is done when the +# parent is locked. +permutation "begin1" "lockexcl1" "reindex2" "drop3" "end1" +permutation "begin1" "lockexcl1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockshare1" "reindex2" "drop3" "end1" +permutation "begin1" "lockshare1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockupdate1" "reindex2" "drop3" "end1" +permutation "begin1" "lockupdate1" "reindex_conc2" "drop3" "end1" + +# An existing partition leaf is dropped after reindex is done when this +# leaf is locked. +permutation "begin1" "lockpartexcl1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartexcl1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockpartshare1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartshare1" "reindex_conc2" "drop3" "end1" +permutation "begin1" "lockpartupdate1" "reindex2" "drop3" "end1" +permutation "begin1" "lockpartupdate1" "reindex_conc2" "drop3" "end1" diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index e3e6634d7e..3ccf3719d4 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2196,18 +2196,6 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 (5 rows) --- REINDEX fails for partitioned indexes -REINDEX INDEX concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes --- REINDEX is a no-op for partitioned tables -REINDEX TABLE concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes to reindex -REINDEX TABLE CONCURRENTLY concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes that can be reindexed concurrently SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; relid | parentrelid | level @@ -2320,6 +2308,152 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 (5 rows) +-- REINDEX for partitioned indexes +-- REINDEX TABLE fails for partitioned indexes +-- Top-most parent index +REINDEX TABLE concur_reindex_part_index; +ERROR: "concur_reindex_part_index" is not a table or materialized view +REINDEX TABLE CONCURRENTLY concur_reindex_part_index; +ERROR: "concur_reindex_part_index" is not a table or materialized view +-- Partitioned index with no leaves +REINDEX TABLE concur_reindex_part_index_10; +ERROR: "concur_reindex_part_index_10" is not a table or materialized view +REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; +ERROR: "concur_reindex_part_index_10" is not a table or materialized view +-- Cannot run in a transaction block +BEGIN; +REINDEX INDEX concur_reindex_part_index; +ERROR: REINDEX INDEX cannot run inside a transaction block +CONTEXT: while reindexing partitioned index "public.concur_reindex_part_index" +ROLLBACK; +-- Helper functions to track changes of relfilenodes in a partition tree. +-- Create a table tracking the relfilenode state. +CREATE OR REPLACE FUNCTION create_relfilenode_part(relname text, indname text) + RETURNS VOID AS + $func$ + BEGIN + EXECUTE format(' + CREATE TABLE %I AS + SELECT oid, relname, relfilenode, relkind, reltoastrelid + FROM pg_class + WHERE oid IN + (SELECT relid FROM pg_partition_tree(''%I''));', + relname, indname); + END + $func$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION compare_relfilenode_part(tabname text) + RETURNS TABLE (relname name, relkind "char", state text) AS + $func$ + BEGIN + RETURN QUERY EXECUTE + format( + 'SELECT b.relname, + b.relkind, + CASE WHEN a.relfilenode = b.relfilenode THEN ''relfilenode is unchanged'' + ELSE ''relfilenode has changed'' END + -- Do not join with OID here as CONCURRENTLY changes it. + FROM %I b JOIN pg_class a ON b.relname = a.relname + ORDER BY 1;', tabname); + END + $func$ LANGUAGE plpgsql; +-- Check that expected relfilenodes are changed, non-concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX INDEX concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- REINDEX for partitioned tables +-- REINDEX INDEX fails for partitioned tables +-- Top-most parent +REINDEX INDEX concur_reindex_part; +ERROR: "concur_reindex_part" is not an index +REINDEX INDEX CONCURRENTLY concur_reindex_part; +ERROR: "concur_reindex_part" is not an index +-- Partitioned with no leaves +REINDEX INDEX concur_reindex_part_10; +ERROR: "concur_reindex_part_10" is not an index +REINDEX INDEX CONCURRENTLY concur_reindex_part_10; +ERROR: "concur_reindex_part_10" is not an index +-- Cannot run in a transaction block +BEGIN; +REINDEX TABLE concur_reindex_part; +ERROR: REINDEX TABLE cannot run inside a transaction block +CONTEXT: while reindexing partitioned table "public.concur_reindex_part" +ROLLBACK; +-- Check that expected relfilenodes are changed, non-concurrent case. +-- Note that the partition tree changes of the *indexes* need to be checked. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX TABLE concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); + create_relfilenode_part +------------------------- + +(1 row) + +REINDEX TABLE CONCURRENTLY concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); + relname | relkind | state +-------------------------------+---------+-------------------------- + concur_reindex_part_index | I | relfilenode is unchanged + concur_reindex_part_index_0 | I | relfilenode is unchanged + concur_reindex_part_index_0_1 | i | relfilenode has changed + concur_reindex_part_index_0_2 | i | relfilenode has changed + concur_reindex_part_index_10 | I | relfilenode is unchanged +(5 rows) + +DROP TABLE reindex_index_status; +DROP FUNCTION create_relfilenode_part; +DROP FUNCTION compare_relfilenode_part; +-- Cleanup of partition tree used for REINDEX test. DROP TABLE concur_reindex_part; -- Check errors -- Cannot run inside a transaction block diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index f3667bacdc..6d98b73365 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -903,12 +903,6 @@ CREATE INDEX concur_reindex_part_index_0_2 ON ONLY concur_reindex_part_0_2 (c1); ALTER INDEX concur_reindex_part_index_0 ATTACH PARTITION concur_reindex_part_index_0_2; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; --- REINDEX fails for partitioned indexes -REINDEX INDEX concur_reindex_part_index_10; -REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; --- REINDEX is a no-op for partitioned tables -REINDEX TABLE concur_reindex_part_10; -REINDEX TABLE CONCURRENTLY concur_reindex_part_10; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; -- REINDEX should preserve dependencies of partition tree. @@ -948,6 +942,88 @@ WHERE classid = 'pg_class'::regclass AND ORDER BY 1, 2; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; + +-- REINDEX for partitioned indexes +-- REINDEX TABLE fails for partitioned indexes +-- Top-most parent index +REINDEX TABLE concur_reindex_part_index; +REINDEX TABLE CONCURRENTLY concur_reindex_part_index; +-- Partitioned index with no leaves +REINDEX TABLE concur_reindex_part_index_10; +REINDEX TABLE CONCURRENTLY concur_reindex_part_index_10; +-- Cannot run in a transaction block +BEGIN; +REINDEX INDEX concur_reindex_part_index; +ROLLBACK; +-- Helper functions to track changes of relfilenodes in a partition tree. +-- Create a table tracking the relfilenode state. +CREATE OR REPLACE FUNCTION create_relfilenode_part(relname text, indname text) + RETURNS VOID AS + $func$ + BEGIN + EXECUTE format(' + CREATE TABLE %I AS + SELECT oid, relname, relfilenode, relkind, reltoastrelid + FROM pg_class + WHERE oid IN + (SELECT relid FROM pg_partition_tree(''%I''));', + relname, indname); + END + $func$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION compare_relfilenode_part(tabname text) + RETURNS TABLE (relname name, relkind "char", state text) AS + $func$ + BEGIN + RETURN QUERY EXECUTE + format( + 'SELECT b.relname, + b.relkind, + CASE WHEN a.relfilenode = b.relfilenode THEN ''relfilenode is unchanged'' + ELSE ''relfilenode has changed'' END + -- Do not join with OID here as CONCURRENTLY changes it. + FROM %I b JOIN pg_class a ON b.relname = a.relname + ORDER BY 1;', tabname); + END + $func$ LANGUAGE plpgsql; +-- Check that expected relfilenodes are changed, non-concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX INDEX concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; + +-- REINDEX for partitioned tables +-- REINDEX INDEX fails for partitioned tables +-- Top-most parent +REINDEX INDEX concur_reindex_part; +REINDEX INDEX CONCURRENTLY concur_reindex_part; +-- Partitioned with no leaves +REINDEX INDEX concur_reindex_part_10; +REINDEX INDEX CONCURRENTLY concur_reindex_part_10; +-- Cannot run in a transaction block +BEGIN; +REINDEX TABLE concur_reindex_part; +ROLLBACK; +-- Check that expected relfilenodes are changed, non-concurrent case. +-- Note that the partition tree changes of the *indexes* need to be checked. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX TABLE concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; +-- concurrent case. +SELECT create_relfilenode_part('reindex_index_status', 'concur_reindex_part_index'); +REINDEX TABLE CONCURRENTLY concur_reindex_part; +SELECT * FROM compare_relfilenode_part('reindex_index_status'); +DROP TABLE reindex_index_status; + +DROP FUNCTION create_relfilenode_part; +DROP FUNCTION compare_relfilenode_part; + +-- Cleanup of partition tree used for REINDEX test. DROP TABLE concur_reindex_part; -- Check errors diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index c16f223e4e..33af4ae02a 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -88,7 +88,9 @@ REINDEX [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] { IN <term><literal>INDEX</literal></term> <listitem> <para> - Recreate the specified index. + Recreate the specified index. This form of <command>REINDEX</command> + cannot be executed inside a transaction block when used with a + partitioned index. </para> </listitem> </varlistentry> @@ -99,6 +101,8 @@ REINDEX [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] { IN <para> Recreate all indexes of the specified table. If the table has a secondary <quote>TOAST</quote> table, that is reindexed as well. + This form of <command>REINDEX</command> cannot be executed inside a + transaction block when used with a partitioned table. </para> </listitem> </varlistentry> @@ -259,8 +263,11 @@ REINDEX [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] { IN </para> <para> - Reindexing partitioned tables or partitioned indexes is not supported. - Each individual partition can be reindexed separately instead. + Reindexing partitioned indexes or partitioned tables is supported + with <command>REINDEX INDEX</command> or <command>REINDEX TABLE</command>, + respectively. Each partition of the specified partitioned relation is + reindexed in a separate transaction. Those commands cannot be used inside + a transaction block when working on a partitioned table or index. </para> <refsect2 id="sql-reindex-concurrently" xreflabel="Rebuilding Indexes Concurrently"> diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 90db550b92..84bc0ee381 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -4044,6 +4044,27 @@ SELECT f_test(100); DROP FUNCTION f_test(int); -- =================================================================== +-- REINDEX +-- =================================================================== +-- remote table is not created here +CREATE FOREIGN TABLE reindex_foreign (c1 int, c2 int) + SERVER loopback2 OPTIONS (table_name 'reindex_local'); +REINDEX TABLE reindex_foreign; -- error +ERROR: "reindex_foreign" is not a table or materialized view +REINDEX TABLE CONCURRENTLY reindex_foreign; -- error +ERROR: "reindex_foreign" is not a table or materialized view +DROP FOREIGN TABLE reindex_foreign; +-- partitions and foreign tables +CREATE TABLE reind_fdw_parent (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE reind_fdw_0_10 PARTITION OF reind_fdw_parent + FOR VALUES FROM (0) TO (10); +CREATE FOREIGN TABLE reind_fdw_10_20 PARTITION OF reind_fdw_parent + FOR VALUES FROM (10) TO (20) + SERVER loopback OPTIONS (table_name 'reind_local_10_20'); +REINDEX TABLE reind_fdw_parent; -- ok +REINDEX TABLE CONCURRENTLY reind_fdw_parent; -- ok +DROP TABLE reind_fdw_parent; +-- =================================================================== -- conversion error -- =================================================================== ALTER FOREIGN TABLE ft1 ALTER COLUMN c8 TYPE int; diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 83971665e3..d452d06343 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -1081,6 +1081,26 @@ $$ LANGUAGE plpgsql; SELECT f_test(100); DROP FUNCTION f_test(int); +-- =================================================================== +-- REINDEX +-- =================================================================== +-- remote table is not created here +CREATE FOREIGN TABLE reindex_foreign (c1 int, c2 int) + SERVER loopback2 OPTIONS (table_name 'reindex_local'); +REINDEX TABLE reindex_foreign; -- error +REINDEX TABLE CONCURRENTLY reindex_foreign; -- error +DROP FOREIGN TABLE reindex_foreign; +-- partitions and foreign tables +CREATE TABLE reind_fdw_parent (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE reind_fdw_0_10 PARTITION OF reind_fdw_parent + FOR VALUES FROM (0) TO (10); +CREATE FOREIGN TABLE reind_fdw_10_20 PARTITION OF reind_fdw_parent + FOR VALUES FROM (10) TO (20) + SERVER loopback OPTIONS (table_name 'reind_local_10_20'); +REINDEX TABLE reind_fdw_parent; -- ok +REINDEX TABLE CONCURRENTLY reind_fdw_parent; -- ok +DROP TABLE reind_fdw_parent; + -- =================================================================== -- conversion error -- =================================================================== diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 3d990463ce..4f7b03630a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2036,6 +2036,7 @@ RegProcedure Regis RegisNode RegisteredBgWorker +ReindexErrorInfo ReindexObjectType ReindexStmt ReindexType -- 2.28.0
signature.asc
Description: PGP signature
