From 6d1d74b2bda66550dda6c64e69ed0222dd188f80 Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Tue, 3 Nov 2020 03:43:53 +0300 Subject: [PATCH] Stopevents --- configure | 34 ++ configure.ac | 9 + src/Makefile.global.in | 1 + src/backend/Makefile | 10 +- src/backend/access/gin/ginget.c | 30 ++ src/backend/storage/buffer/bufmgr.c | 29 ++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/lmgr/.gitignore | 2 + src/backend/storage/lmgr/Makefile | 10 +- .../storage/lmgr/generate-stopeventnames.pl | 62 +++ src/backend/storage/lmgr/stopevent.c | 372 ++++++++++++++++++ src/backend/storage/lmgr/stopeventnames.txt | 2 + src/backend/utils/activity/wait_event.c | 3 + src/backend/utils/adt/lockfuncs.c | 4 + src/backend/utils/misc/guc.c | 23 ++ src/include/catalog/pg_proc.dat | 13 + src/include/pg_config.h.in | 3 + src/include/storage/.gitignore | 1 + src/include/storage/stopevent.h | 37 ++ src/include/utils/wait_event.h | 1 + src/test/isolation/Makefile | 14 +- .../expected/gin-traverse-deleted-pages.out | 26 ++ .../isolation_schedule_with_stop_events | 94 +++++ .../specs/gin-traverse-deleted-pages.spec | 30 ++ src/tools/pgindent/typedefs.list | 1 + 25 files changed, 808 insertions(+), 6 deletions(-) create mode 100644 src/backend/storage/lmgr/generate-stopeventnames.pl create mode 100644 src/backend/storage/lmgr/stopevent.c create mode 100644 src/backend/storage/lmgr/stopeventnames.txt create mode 100644 src/include/storage/stopevent.h create mode 100644 src/test/isolation/expected/gin-traverse-deleted-pages.out create mode 100644 src/test/isolation/isolation_schedule_with_stop_events create mode 100644 src/test/isolation/specs/gin-traverse-deleted-pages.spec diff --git a/configure b/configure index a268780c5db..e0ad4a166f1 100755 --- a/configure +++ b/configure @@ -699,6 +699,7 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +enable_stop_events ZSTD_LIBS ZSTD_CFLAGS with_zstd @@ -875,6 +876,7 @@ with_system_tzdata with_zlib with_lz4 with_zstd +enable_stop_events with_gnu_ld with_ssl with_openssl @@ -1541,6 +1543,7 @@ Optional Features: --enable-depend turn on automatic dependency tracking --enable-cassert enable assertion checks (for debugging) --disable-thread-safety disable thread-safety in client libraries + --enable-stop-events enable stop events (for debugging) --disable-largefile omit support for large files Optional Packages: @@ -9249,6 +9252,37 @@ fi esac done fi + +# +# Stop events +# + + +# Check whether --enable-stop-events was given. +if test "${enable_stop_events+set}" = set; then : + enableval=$enable_stop_events; + case $enableval in + yes) + +$as_echo "#define USE_STOP_EVENTS 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-stop-events option" "$LINENO" 5 + ;; + esac + +else + enable_stop_events=no + +fi + + + + # # Assignments # diff --git a/configure.ac b/configure.ac index 993b5d5cb0a..c9dc768ad60 100644 --- a/configure.ac +++ b/configure.ac @@ -1094,6 +1094,15 @@ if test "$with_zstd" = yes; then esac done fi + +# +# Stop events +# +PGAC_ARG_BOOL(enable, stop-events, no, [enable stop events (for debugging)], + [AC_DEFINE([USE_STOP_EVENTS], 1, + [Define to 1 to build with stop events. (--enable-stop-events)])]) +AC_SUBST(enable_stop_events) + # # Assignments # diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 5664c645f82..d46ad160a61 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -203,6 +203,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_tap_tests = @enable_tap_tests@ enable_thread_safety = @enable_thread_safety@ +enable_stop_events = @enable_stop_events@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ diff --git a/src/backend/Makefile b/src/backend/Makefile index f498cfd5930..050656aa9d9 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -126,6 +126,9 @@ parser/gram.h: parser/gram.y storage/lmgr/lwlocknames.h: storage/lmgr/generate-lwlocknames.pl storage/lmgr/lwlocknames.txt $(MAKE) -C storage/lmgr lwlocknames.h lwlocknames.c +storage/lmgr/stopeventnames.h: storage/lmgr/generate-stopeventnames.pl storage/lmgr/stopeventnames.txt + $(MAKE) -C storage/lmgr stopeventnames.h stopeventnames.c + # run this unconditionally to avoid needing to know its dependencies here: submake-catalog-headers: $(MAKE) -C catalog distprep generated-header-symlinks @@ -153,7 +156,7 @@ submake-utils-headers: .PHONY: generated-headers -generated-headers: $(top_builddir)/src/include/parser/gram.h $(top_builddir)/src/include/storage/lwlocknames.h submake-catalog-headers submake-nodes-headers submake-utils-headers +generated-headers: $(top_builddir)/src/include/parser/gram.h $(top_builddir)/src/include/storage/lwlocknames.h $(top_builddir)/src/include/storage/stopeventnames.h submake-catalog-headers submake-nodes-headers submake-utils-headers $(top_builddir)/src/include/parser/gram.h: parser/gram.h prereqdir=`cd '$(dir $<)' >/dev/null && pwd` && \ @@ -165,6 +168,11 @@ $(top_builddir)/src/include/storage/lwlocknames.h: storage/lmgr/lwlocknames.h cd '$(dir $@)' && rm -f $(notdir $@) && \ $(LN_S) "$$prereqdir/$(notdir $<)" . +$(top_builddir)/src/include/storage/stopeventnames.h: storage/lmgr/stopeventnames.h + prereqdir=`cd '$(dir $<)' >/dev/null && pwd` && \ + cd '$(dir $@)' && rm -f $(notdir $@) && \ + $(LN_S) "$$prereqdir/$(notdir $<)" . + utils/probes.o: utils/probes.d $(SUBDIROBJS) $(DTRACE) $(DTRACEFLAGS) -C -G -s $(call expand_subsys,$^) -o $@ diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c index fc85ba99ac1..5e170e80fda 100644 --- a/src/backend/access/gin/ginget.c +++ b/src/backend/access/gin/ginget.c @@ -19,7 +19,9 @@ #include "common/pg_prng.h" #include "miscadmin.h" #include "storage/predicate.h" +#include "storage/stopevent.h" #include "utils/datum.h" +#include "utils/jsonb.h" #include "utils/memutils.h" #include "utils/rel.h" @@ -643,6 +645,29 @@ startScan(IndexScanDesc scan) startScanKey(ginstate, so, so->keys + i); } +#ifdef USE_STOP_EVENTS +static Jsonb * +EntryFindPostingLeafPageStopEventParams(Relation index, + OffsetNumber attnum, + BlockNumber rootBlockNum) +{ + MemoryContext oldCxt; + JsonbParseState *state = NULL; + Jsonb *res; + + stopevents_make_cxt(); + oldCxt = MemoryContextSwitchTo(stopevents_cxt); + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + relation_stopevent_params(&state, index); + jsonb_push_int8_key(&state, "attnum", attnum); + jsonb_push_int8_key(&state, "rootBlockNum", rootBlockNum); + res = JsonbValueToJsonb(pushJsonbValue(&state, WJB_END_OBJECT, NULL)); + MemoryContextSwitchTo(oldCxt); + + return res; +} +#endif + /* * Load the next batch of item pointers from a posting tree. * @@ -696,6 +721,11 @@ entryLoadMoreItems(GinState *ginstate, GinScanEntry entry, OffsetNumberNext(GinItemPointerGetOffsetNumber(&advancePast))); } entry->btree.fullScan = false; + + STOPEVENT(EntryFindPostingLeafPageStopEvent, + EntryFindPostingLeafPageStopEventParams(ginstate->index, + entry->attnum, + entry->btree.rootBlkno)); stack = ginFindLeafPage(&entry->btree, true, false, snapshot); /* we don't need the stack, just the buffer. */ diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e898ffad7bb..210f1967576 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -51,6 +51,7 @@ #include "storage/proc.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "storage/stopevent.h" #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" @@ -1629,6 +1630,29 @@ MarkBufferDirty(Buffer buffer) } } +#ifdef USE_STOP_EVENTS +static Jsonb * +ReleaseAndReadBufferStopEventParams(Relation relation, + BlockNumber oldBlockNum, + BlockNumber newBlockNum) +{ + MemoryContext oldCxt; + JsonbParseState *state = NULL; + Jsonb *res; + + stopevents_make_cxt(); + oldCxt = MemoryContextSwitchTo(stopevents_cxt); + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + relation_stopevent_params(&state, relation); + jsonb_push_int8_key(&state, "oldBlockNum", oldBlockNum); + jsonb_push_int8_key(&state, "newBlockNum", newBlockNum); + res = JsonbValueToJsonb(pushJsonbValue(&state, WJB_END_OBJECT, NULL)); + MemoryContextSwitchTo(oldCxt); + + return res; +} +#endif + /* * ReleaseAndReadBuffer -- combine ReleaseBuffer() and ReadBuffer() * @@ -1649,9 +1673,11 @@ ReleaseAndReadBuffer(Buffer buffer, { ForkNumber forkNum = MAIN_FORKNUM; BufferDesc *bufHdr; + BlockNumber oldBlockNum = InvalidBlockNumber; if (BufferIsValid(buffer)) { + oldBlockNum = BufferGetBlockNumber(buffer); Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) { @@ -1675,6 +1701,9 @@ ReleaseAndReadBuffer(Buffer buffer, } } + STOPEVENT(ReleaseAndReadBufferStopEvent, + ReleaseAndReadBufferStopEventParams(relation, oldBlockNum, blockNum)); + return ReadBuffer(relation, blockNum); } diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 1a6f5270518..c3cbb6e35e3 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -47,6 +47,7 @@ #include "storage/procsignal.h" #include "storage/sinvaladt.h" #include "storage/spin.h" +#include "storage/stopevent.h" #include "utils/snapmgr.h" /* GUCs */ @@ -141,6 +142,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, SyncScanShmemSize()); size = add_size(size, AsyncShmemSize()); size = add_size(size, StatsShmemSize()); + size = add_size(size, StopEventShmemSize()); #ifdef EXEC_BACKEND size = add_size(size, ShmemBackendArraySize()); #endif @@ -293,6 +295,7 @@ CreateSharedMemoryAndSemaphores(void) SyncScanShmemInit(); AsyncShmemInit(); StatsShmemInit(); + StopEventShmemInit(); #ifdef EXEC_BACKEND diff --git a/src/backend/storage/lmgr/.gitignore b/src/backend/storage/lmgr/.gitignore index dab4c3f5806..c035a2e80ab 100644 --- a/src/backend/storage/lmgr/.gitignore +++ b/src/backend/storage/lmgr/.gitignore @@ -1,3 +1,5 @@ /lwlocknames.c /lwlocknames.h /s_lock_test +/stopeventnames.c +/stopeventnames.h diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index b25b7ee421d..314a0727ec7 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -22,7 +22,9 @@ OBJS = \ predicate.o \ proc.o \ s_lock.o \ - spin.o + spin.o \ + stopevent.o \ + stopeventnames.o include $(top_srcdir)/src/backend/common.mk @@ -42,6 +44,12 @@ lwlocknames.c: lwlocknames.h lwlocknames.h: $(top_srcdir)/src/backend/storage/lmgr/lwlocknames.txt generate-lwlocknames.pl $(PERL) $(srcdir)/generate-lwlocknames.pl $< +stopeventnames.c: stopeventnames.h + touch $@ + +stopeventnames.h: $(top_srcdir)/src/backend/storage/lmgr/stopeventnames.txt generate-stopeventnames.pl + $(PERL) $(srcdir)/generate-stopeventnames.pl $< + check: s_lock_test ./s_lock_test diff --git a/src/backend/storage/lmgr/generate-stopeventnames.pl b/src/backend/storage/lmgr/generate-stopeventnames.pl new file mode 100644 index 00000000000..8fbf1f54dee --- /dev/null +++ b/src/backend/storage/lmgr/generate-stopeventnames.pl @@ -0,0 +1,62 @@ +#!/usr/bin/perl +# +# Generate stopeventnames.h and stopeventnames.c from stopeventnames.txt +# Copyright (c) 2020, PostgreSQL Global Development Group + +use strict; +use warnings; + +my $continue = "\n"; + +open my $stopeventnames, '<', $ARGV[0] or die; + +# Include PID in suffix in case parallel make runs this multiple times. +my $htmp = "stopeventnames.h.tmp$$"; +my $ctmp = "stopeventnames.c.tmp$$"; +open my $h, '>', $htmp or die "Could not open $htmp: $!"; +open my $c, '>', $ctmp or die "Could not open $ctmp: $!"; + +my $autogen = + "/* autogenerated from src/backend/storage/lmgr/stopeventnames.txt, do not edit */\n"; +print $h $autogen; +print $h "/* there is deliberately not an #ifndef STOPEVENTNAMES_H here */\n\n"; +print $c $autogen, "\n"; + +print $c "const char *const stopeventnames[] = {"; + +my $eventidx = 0; +while (<$stopeventnames>) +{ + chomp; + + # Skip comments + next if /^#/; + next if /^\s*$/; + + die "unable to parse stopeventnames.txt" + unless /^(\w+)$/; + + my $eventname = $1; + + my $trimmedeventname = $eventname; + $trimmedeventname =~ s/StopEvent$//; + die "event names must end with 'StopEvent'" if $trimmedeventname eq $eventname; + + printf $c "%s \"%s\"", $continue, $trimmedeventname; + $continue = ",\n"; + + print $h "#define $eventname $eventidx\n"; + $eventidx++; +} + +printf $c "\n};\n"; +print $h "\n"; +printf $h "#define NUM_BUILTIN_STOPEVENTS %s\n", $eventidx; + +close $h; +close $c; + +rename($htmp, 'stopeventnames.h') || die "rename: $htmp: $!"; +rename($ctmp, 'stopeventnames.c') || die "rename: $ctmp: $!"; + +close $stopeventnames; diff --git a/src/backend/storage/lmgr/stopevent.c b/src/backend/storage/lmgr/stopevent.c new file mode 100644 index 00000000000..6419356c8fb --- /dev/null +++ b/src/backend/storage/lmgr/stopevent.c @@ -0,0 +1,372 @@ +/*------------------------------------------------------------------------- + * + * stopevent.c + * Auxiliary infrastructure for automated testing of concurrency issues + * + * Copyright (c) 2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/storage/lmgr/stopevent.c +*/ +#include "postgres.h" + +#include "commands/dbcommands.h" +#include "miscadmin.h" +#include "nodes/execnodes.h" +#include "pgstat.h" +#include "storage/condition_variable.h" +#include "storage/proclist.h" +#include "storage/shmem.h" +#include "storage/stopevent.h" +#include "utils/builtins.h" +#include "utils/jsonpath.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +#define QUERY_BUFFER_SIZE 1024 + +typedef struct +{ + char condition[QUERY_BUFFER_SIZE]; + bool enabled; + slock_t lock; + ConditionVariable cv; +} StopEvent; + +bool enable_stopevents = false; +bool trace_stopevents = false; +StopEvent *stopevents = NULL; +MemoryContext stopevents_cxt = NULL; + +Size +StopEventShmemSize(void) +{ +#ifdef USE_STOP_EVENTS + Size size; + + size = mul_size(NUM_BUILTIN_STOPEVENTS, sizeof(StopEvent)); + return size; +#else + return 0; +#endif +} + +void +StopEventShmemInit(void) +{ +#ifndef USE_STOP_EVENTS + Size size = StopEventShmemSize(); + bool found; + + stopevents = (StopEvent *) ShmemInitStruct("Stop events Data", + size, + &found); + + if (!found) + { + int i; + + for (i = 0; i < NUM_BUILTIN_STOPEVENTS; i++) + { + SpinLockInit(&stopevents[i].lock); + stopevents[i].enabled = false; + ConditionVariableInit(&stopevents[i].cv); + } + } +#else + return; +#endif +} + +static StopEvent * +find_stop_event(text *name) +{ + int i; + char *name_data = VARDATA_ANY(name); + int len = VARSIZE_ANY_EXHDR(name); + + for (i = 0; i < NUM_BUILTIN_STOPEVENTS; i++) + { + if (strlen(stopeventnames[i]) == len && + memcmp(name_data, stopeventnames[i], len) == 0) + return &stopevents[i]; + } + + elog(ERROR, "unknown stop event: \"%s\"", text_to_cstring(name)); + return NULL; +} + +#ifdef USE_STOP_EVENTS +#define CHECK_FOR_STOPEVENTS() +#else +#define CHECK_FOR_STOPEVENTS() \ + ereport(ERROR, \ + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ + errmsg("stop events are not supported"), \ + errhint("Try to rebuild PostgreSQL with --enable-stop-events flag."))) +#endif + +Datum +pg_stopevent_set(PG_FUNCTION_ARGS) +{ + text *event_name = PG_GETARG_TEXT_PP(0); + JsonPath *condition = PG_GETARG_JSONPATH_P(1); + StopEvent *event; + + CHECK_FOR_STOPEVENTS(); + + event = find_stop_event(event_name); + + if (VARSIZE_ANY(condition) > QUERY_BUFFER_SIZE) + elog(ERROR, "jsonpath condition is too long"); + + SpinLockAcquire(&event->lock); + event->enabled = true; + memcpy(&event->condition, condition, VARSIZE_ANY(condition)); + SpinLockRelease(&event->lock); + + ConditionVariableBroadcast(&event->cv); + + PG_FREE_IF_COPY(condition, 1); + PG_RETURN_VOID(); +} + +Datum +pg_stopevent_reset(PG_FUNCTION_ARGS) +{ + text *event_name = PG_GETARG_TEXT_PP(0); + StopEvent *event; + + CHECK_FOR_STOPEVENTS(); + + event = find_stop_event(event_name); + + SpinLockAcquire(&event->lock); + event->enabled = false; + SpinLockRelease(&event->lock); + + ConditionVariableBroadcast(&event->cv); + + PG_RETURN_VOID(); +} + +Datum +pg_stopevents(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + bool randomAccess; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext oldcontext; + AttrNumber attnum; + int i; + + CHECK_FOR_STOPEVENTS(); + + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ + oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); + + tupdesc = CreateTemplateTupleDesc(3); + attnum = (AttrNumber) 1; + TupleDescInitEntry(tupdesc, attnum, "stopevent", TEXTOID, -1, 0); + attnum++; + TupleDescInitEntry(tupdesc, attnum, "condition", JSONPATHOID, -1, 0); + attnum++; + TupleDescInitEntry(tupdesc, attnum, "waiters", INT4ARRAYOID, -1, 0); + + randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + for (i = 0; i < NUM_BUILTIN_STOPEVENTS; i++) + { + Datum values[3]; + bool nulls[3] = {false, false, false}; + StopEvent *event = &stopevents[i]; + proclist_mutable_iter iter; + List *waiters = NIL; + Datum *elems; + ListCell *lc; + int j; + + SpinLockAcquire(&event->lock); + if (!event->enabled) + { + SpinLockRelease(&event->lock); + continue; + } + values[0] = PointerGetDatum(cstring_to_text(stopeventnames[i])); + values[1] = PointerGetDatum(&event->condition); + + SpinLockAcquire(&event->cv.mutex); + proclist_foreach_modify(iter, &event->cv.wakeup, cvWaitLink) + { + PGPROC *waiter = GetPGProcByNumber(iter.cur); + + waiters = lappend_int(waiters, waiter->pid); + } + SpinLockRelease(&event->cv.mutex); + + elems = (Datum *) palloc(sizeof(Datum) * list_length(waiters)); + j = 0; + foreach(lc, waiters) + { + elems[j] = Int32GetDatum(lfirst_int(lc)); + j++; + } + values[2] = PointerGetDatum(construct_array(elems, list_length(waiters), INT4OID, 4, true, 'i')); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + SpinLockRelease(&event->lock); + } + PG_RETURN_VOID(); +} + +bool +pid_is_waiting_for_stopevent(int pid) +{ + int i; + + for (i = 0; i < NUM_BUILTIN_STOPEVENTS; i++) + { + StopEvent *event = &stopevents[i]; + proclist_mutable_iter iter; + + SpinLockAcquire(&event->lock); + if (!event->enabled) + { + SpinLockRelease(&event->lock); + continue; + } + + SpinLockAcquire(&event->cv.mutex); + proclist_foreach_modify(iter, &event->cv.wakeup, cvWaitLink) + { + PGPROC *waiter = GetPGProcByNumber(iter.cur); + + if (waiter->pid == pid) + { + SpinLockRelease(&event->cv.mutex); + SpinLockRelease(&event->lock); + return true; + } + } + SpinLockRelease(&event->cv.mutex); + SpinLockRelease(&event->lock); + } + return false; +} + +static bool +check_stopevent_condition(StopEvent *event, Jsonb *params) +{ + Datum res; + + SpinLockAcquire(&event->lock); + if (!event->enabled) + { + SpinLockRelease(&event->lock); + return false; + } + + res = DirectFunctionCall2(jsonb_path_match, + PointerGetDatum(params), + PointerGetDatum(&event->condition)); + + SpinLockRelease(&event->lock); + + return DatumGetBool(res); +} + +void +handle_stopevent(int event_id, Jsonb *params) +{ + StopEvent *event = &stopevents[event_id]; + + Assert(event_id >= 0 && event_id < NUM_BUILTIN_STOPEVENTS); + + if (event->enabled && check_stopevent_condition(event, params)) + { + ConditionVariablePrepareToSleep(&event->cv); + for (;;) + { + if (!check_stopevent_condition(event, params)) + break; + ConditionVariableSleep(&event->cv, WAIT_EVENT_STOPEVENT); + } + ConditionVariableCancelSleep(); + } + + if (trace_stopevents) + { + char *params_string; + + params_string = DatumGetCString(DirectFunctionCall1(jsonb_out, PointerGetDatum(params))); + elog(DEBUG2, "stop event \"%s\", params \"%s\"", + stopeventnames[event_id], + params_string); + pfree(params_string); + } + + MemoryContextReset(stopevents_cxt); +} + +void +stopevents_make_cxt(void) +{ + if (!stopevents_cxt) + stopevents_cxt = AllocSetContextCreate(TopMemoryContext, + "CacheMemoryContext", + ALLOCSET_DEFAULT_SIZES); +} + +void +jsonb_push_key(JsonbParseState **state, char *key) +{ + JsonbValue jval; + + jval.type = jbvString; + jval.val.string.len = strlen(key); + jval.val.string.val = key; + (void) pushJsonbValue(state, WJB_KEY, &jval); +} + +void +jsonb_push_int8_key(JsonbParseState **state, char *key, int64 value) +{ + JsonbValue jval; + + jsonb_push_key(state, key); + + jval.type = jbvNumeric; + jval.val.numeric = DatumGetNumeric(DirectFunctionCall1(int8_numeric, Int64GetDatum(value))); + (void) pushJsonbValue(state, WJB_VALUE, &jval); + +} + +void +jsonb_push_string_key(JsonbParseState **state, const char *key, + const char *value) +{ + JsonbValue jval; + + jsonb_push_key(state, (char *) key); + + jval.type = jbvString; + jval.val.string.len = strlen(value); + jval.val.string.val = (char *) value; + (void) pushJsonbValue(state, WJB_VALUE, &jval); +} + +void +relation_stopevent_params(JsonbParseState **state, Relation relation) +{ + jsonb_push_int8_key(state, "datoid", MyDatabaseId); + jsonb_push_string_key(state, "datname", get_database_name(MyDatabaseId)); + jsonb_push_int8_key(state, "reloid", relation->rd_id); + jsonb_push_string_key(state, "relname", NameStr(relation->rd_rel->relname)); +} diff --git a/src/backend/storage/lmgr/stopeventnames.txt b/src/backend/storage/lmgr/stopeventnames.txt new file mode 100644 index 00000000000..b5330a1d13f --- /dev/null +++ b/src/backend/storage/lmgr/stopeventnames.txt @@ -0,0 +1,2 @@ +ReleaseAndReadBufferStopEvent +EntryFindPostingLeafPageStopEvent diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 92f24a6c9bc..ab82f754bf2 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -448,6 +448,9 @@ pgstat_get_wait_ipc(WaitEventIPC w) case WAIT_EVENT_SAFE_SNAPSHOT: event_name = "SafeSnapshot"; break; + case WAIT_EVENT_STOPEVENT: + event_name = "StopEvent"; + break; case WAIT_EVENT_SYNC_REP: event_name = "SyncRep"; break; diff --git a/src/backend/utils/adt/lockfuncs.c b/src/backend/utils/adt/lockfuncs.c index f9b324efec7..121d37aea2e 100644 --- a/src/backend/utils/adt/lockfuncs.c +++ b/src/backend/utils/adt/lockfuncs.c @@ -18,6 +18,7 @@ #include "funcapi.h" #include "miscadmin.h" #include "storage/predicate_internals.h" +#include "storage/stopevent.h" #include "utils/array.h" #include "utils/builtins.h" @@ -650,6 +651,9 @@ pg_isolation_test_session_is_blocked(PG_FUNCTION_ARGS) if (GetSafeSnapshotBlockingPids(blocked_pid, &dummy, 1) > 0) PG_RETURN_BOOL(true); + if (pid_is_waiting_for_stopevent(blocked_pid)) + PG_RETURN_BOOL(true); + PG_RETURN_BOOL(false); } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 9fbbfb1be54..39b72321eca 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -93,6 +93,7 @@ #include "storage/predicate.h" #include "storage/proc.h" #include "storage/standby.h" +#include "storage/stopevent.h" #include "tcop/tcopprot.h" #include "tsearch/ts_cache.h" #include "utils/acl.h" @@ -2184,6 +2185,28 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"enable_stopevents", PGC_SUSET, DEVELOPER_OPTIONS, + gettext_noop("Sets whether stop events are enabled."), + NULL, + GUC_NOT_IN_SAMPLE + }, + &enable_stopevents, + false, + NULL, NULL, NULL + }, + + { + {"trace_stopevents", PGC_SUSET, DEVELOPER_OPTIONS, + gettext_noop("Sets whether trace stop events to the log."), + NULL, + GUC_NOT_IN_SAMPLE + }, + &trace_stopevents, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index be47583122b..92ba9755b45 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11894,4 +11894,17 @@ prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary', prosrc => 'brin_minmax_multi_summary_send' }, +{ oid => '8000', descr => 'set stop event', + proname => 'pg_stopevent_set', prorettype => 'void', proargtypes => 'text jsonpath', + prosrc => 'pg_stopevent_set', provolatile => 'v' }, + +{ oid => '8001', descr => 'reset stop event', + proname => 'pg_stopevent_reset', prorettype => 'void', proargtypes => 'text', + prosrc => 'pg_stopevent_reset', provolatile => 'v' }, + +{ oid => '8002', descr => 'view stop events', + proname => 'pg_stopevents', prorows => '10', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => '', + proallargtypes => '{text,jsonpath,_int4}', proargmodes => '{o,o,o}', + proargnames => '{event_name,condition,waiters}', prosrc => 'pg_stopevents' }, ] diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index c5a80b829e7..d897031b4b0 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -739,6 +739,9 @@ /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK +/* Define to 1 to build with stop events. (--enable-stop-events) */ +#undef USE_STOP_EVENTS + /* Define to build with systemd support. (--with-systemd) */ #undef USE_SYSTEMD diff --git a/src/include/storage/.gitignore b/src/include/storage/.gitignore index 209c8be7223..a1e65d8df4e 100644 --- a/src/include/storage/.gitignore +++ b/src/include/storage/.gitignore @@ -1 +1,2 @@ /lwlocknames.h +/stopeventnames.h diff --git a/src/include/storage/stopevent.h b/src/include/storage/stopevent.h new file mode 100644 index 00000000000..b6cafbe5bab --- /dev/null +++ b/src/include/storage/stopevent.h @@ -0,0 +1,37 @@ +#ifndef SRC_STOPEVENT_H +#define SRC_STOPEVENT_H + +#include "utils/jsonb.h" +#include "storage/stopeventnames.h" + +extern bool enable_stopevents; +extern bool trace_stopevents; +extern const char *const stopeventnames[]; +extern MemoryContext stopevents_cxt; + +#ifdef USE_STOP_EVENTS +#define STOPEVENT(event_id, params) \ + do { \ + if (enable_stopevents || trace_stopevents) \ + handle_stopevent((event_id), (params)); \ + } while(0) +#else +#define STOPEVENT(event_id, params) +#endif + +extern Size StopEventShmemSize(void); +extern void StopEventShmemInit(void); +extern Datum pg_stopevent_set(PG_FUNCTION_ARGS); +extern Datum pg_stopevent_reset(PG_FUNCTION_ARGS); +extern Datum pg_stopevents(PG_FUNCTION_ARGS); +extern bool pid_is_waiting_for_stopevent(int pid); +extern void handle_stopevent(int event_id, Jsonb *params); +extern void stopevents_make_cxt(void); +extern void jsonb_push_key(JsonbParseState **state, char *key); +extern void jsonb_push_int8_key(JsonbParseState **state, char *key, int64 value); +extern void jsonb_push_string_key(JsonbParseState **state, const char *key, + const char *value); +extern void relation_stopevent_params(JsonbParseState **state, + Relation relation); + +#endif /* SRC_STOPEVENT_H */ diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index 6f2d5612e06..6b2f4b2d2d9 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -125,6 +125,7 @@ typedef enum WAIT_EVENT_REPLICATION_SLOT_DROP, WAIT_EVENT_RESTORE_COMMAND, WAIT_EVENT_SAFE_SNAPSHOT, + WAIT_EVENT_STOPEVENT, WAIT_EVENT_SYNC_REP, WAIT_EVENT_WAL_RECEIVER_EXIT, WAIT_EVENT_WAL_RECEIVER_WAIT_START, diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index 0d452c89d40..2bd361b26af 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -17,6 +17,12 @@ OBJS = \ isolationtester.o \ specparse.o +ifeq ($(enable_stop-events),yes) +schedule = $(srcdir)/isolation_schedule_with_stop_events +else +schedule = $(srcdir)/isolation_schedule +endif + all: isolationtester$(X) pg_isolation_regress$(X) install: all installdirs @@ -58,16 +64,16 @@ maintainer-clean: distclean rm -f specparse.c specscanner.c installcheck: all - $(pg_isolation_regress_installcheck) --schedule=$(srcdir)/isolation_schedule + $(pg_isolation_regress_installcheck) --schedule=$(schedule) check: all - $(pg_isolation_regress_check) --schedule=$(srcdir)/isolation_schedule + $(pg_isolation_regress_check) --schedule=$(schedule) # Non-default tests. It only makes sense to run these if set up to use # prepared transactions, via TEMP_CONFIG for the check case, or via the # postgresql.conf for the installcheck case. installcheck-prepared-txns: all temp-install - $(pg_isolation_regress_installcheck) --schedule=$(srcdir)/isolation_schedule prepared-transactions prepared-transactions-cic + $(pg_isolation_regress_installcheck) --schedule=$(schedule) prepared-transactions prepared-transactions-cic check-prepared-txns: all temp-install - $(pg_isolation_regress_check) --schedule=$(srcdir)/isolation_schedule prepared-transactions prepared-transactions-cic + $(pg_isolation_regress_check) --schedule=$(schedule) prepared-transactions prepared-transactions-cic diff --git a/src/test/isolation/expected/gin-traverse-deleted-pages.out b/src/test/isolation/expected/gin-traverse-deleted-pages.out new file mode 100644 index 00000000000..18d86b95db2 --- /dev/null +++ b/src/test/isolation/expected/gin-traverse-deleted-pages.out @@ -0,0 +1,26 @@ +Parsed test spec with 2 sessions + +starting permutation: s2s1 s1s s2s2 s2v s2s3 +step s2s1: select pg_stopevent_set('EntryFindPostingLeafPage', + '$.datname == "isolation_regression" && $.relname == "tmp_idx"'); +pg_stopevent_set + + +step s1s: select * from tmp where ar @> array[1,2]; +step s2s2: select pg_stopevent_set('ReleaseAndReadBuffer', + '$.datname == "isolation_regression" && $.relname == "tmp_idx"'); + select pg_stopevent_reset('EntryFindPostingLeafPage'); +pg_stopevent_set + + +pg_stopevent_reset + + +step s2v: vacuum tmp; +step s2s3: select pg_stopevent_reset('ReleaseAndReadBuffer'); +pg_stopevent_reset + + +step s1s: <... completed> +ar + diff --git a/src/test/isolation/isolation_schedule_with_stop_events b/src/test/isolation/isolation_schedule_with_stop_events new file mode 100644 index 00000000000..2e7063168be --- /dev/null +++ b/src/test/isolation/isolation_schedule_with_stop_events @@ -0,0 +1,94 @@ +test: read-only-anomaly +test: read-only-anomaly-2 +test: read-only-anomaly-3 +test: read-write-unique +test: read-write-unique-2 +test: read-write-unique-3 +test: read-write-unique-4 +test: simple-write-skew +test: receipt-report +test: temporal-range-integrity +test: project-manager +test: classroom-scheduling +test: total-cash +test: referential-integrity +test: ri-trigger +test: partial-index +test: two-ids +test: multiple-row-versions +test: index-only-scan +test: predicate-lock-hot-tuple +test: update-conflict-out +test: deadlock-simple +test: deadlock-hard +test: deadlock-soft +test: deadlock-soft-2 +test: deadlock-parallel +test: fk-contention +test: fk-deadlock +test: fk-deadlock2 +test: fk-partitioned-1 +test: fk-partitioned-2 +test: eval-plan-qual +test: eval-plan-qual-trigger +test: lock-update-delete +test: lock-update-traversal +test: inherit-temp +test: insert-conflict-do-nothing +test: insert-conflict-do-nothing-2 +test: insert-conflict-do-update +test: insert-conflict-do-update-2 +test: insert-conflict-do-update-3 +test: insert-conflict-specconflict +test: delete-abort-savept +test: delete-abort-savept-2 +test: aborted-keyrevoke +test: multixact-no-deadlock +test: multixact-no-forget +test: lock-committed-update +test: lock-committed-keyupdate +test: update-locked-tuple +test: reindex-concurrently +test: reindex-schema +test: propagate-lock-delete +test: tuplelock-conflict +test: tuplelock-update +test: tuplelock-upgrade-no-deadlock +test: freeze-the-dead +test: nowait +test: nowait-2 +test: nowait-3 +test: nowait-4 +test: nowait-5 +test: skip-locked +test: skip-locked-2 +test: skip-locked-3 +test: skip-locked-4 +test: drop-index-concurrently-1 +test: multiple-cic +test: alter-table-1 +test: alter-table-2 +test: alter-table-3 +test: alter-table-4 +test: create-trigger +test: sequence-ddl +test: async-notify +test: vacuum-reltuples +test: timeouts +test: vacuum-concurrent-drop +test: vacuum-conflict +test: vacuum-skip-locked +test: horizons +test: predicate-hash +test: predicate-gist +test: predicate-gin +test: partition-concurrent-attach +test: partition-key-update-1 +test: partition-key-update-2 +test: partition-key-update-3 +test: partition-key-update-4 +test: plpgsql-toast +test: truncate-conflict +test: serializable-parallel +test: serializable-parallel-2 +test: gin-traverse-deleted-pages diff --git a/src/test/isolation/specs/gin-traverse-deleted-pages.spec b/src/test/isolation/specs/gin-traverse-deleted-pages.spec new file mode 100644 index 00000000000..5814cec8354 --- /dev/null +++ b/src/test/isolation/specs/gin-traverse-deleted-pages.spec @@ -0,0 +1,30 @@ +setup +{ + create table tmp (ar int[]) with (autovacuum_enabled = false); + insert into tmp (select array[1] from generate_series(1,10000) i); + insert into tmp values (array[1,2]); + insert into tmp (select array[1] from generate_series(1,10000) i); + create index tmp_idx on tmp using gin(ar); + delete from tmp; +} + +teardown +{ + DROP TABLE tmp; +} + +session "s1" +setup { set enable_stopevents = true; + set max_parallel_workers_per_gather = 0; } +step "s1s" { select * from tmp where ar @> array[1,2]; } + +session "s2" +step "s2s1" { select pg_stopevent_set('EntryFindPostingLeafPage', + '$.datname == "isolation_regression" && $.relname == "tmp_idx"'); } +step "s2v" { vacuum tmp; } +step "s2s2" { select pg_stopevent_set('ReleaseAndReadBuffer', + '$.datname == "isolation_regression" && $.relname == "tmp_idx"'); + select pg_stopevent_reset('EntryFindPostingLeafPage'); } +step "s2s3" { select pg_stopevent_reset('ReleaseAndReadBuffer'); } + +permutation "s2s1" "s1s" "s2s2" "s2v" "s2s3" diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 8ad112c44d0..08634ef9392 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2612,6 +2612,7 @@ StdAnalyzeData StdRdOptIndexCleanup StdRdOptions Step +StopEvent StopList StrategyNumber StreamCtl -- 2.24.3 (Apple Git-128)