From 41c08290d25eebc5e2aad9e6211db4bc7a9053b4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 22 Nov 2019 10:12:22 -0500 Subject: [PATCH v10 3/3] tableam: New callback relation_fetch_toast_slice. Instead of always calling heap_fetch_toast_slice during detoasting, invoke a table AM callback which, when the toast table is a heap table, will be heap_fetch_toast_slice. This makes it possible for a table AM other than heap to be used as a TOAST table. --- src/backend/access/common/detoast.c | 199 +---------------------- src/backend/access/heap/heapam_handler.c | 1 + src/backend/access/heap/heaptoast.c | 185 +++++++++++++++++++++ src/include/access/heaptoast.h | 10 ++ src/include/access/tableam.h | 46 ++++++ 5 files changed, 248 insertions(+), 193 deletions(-) diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c index c1e6eabe89..3df0cf2512 100644 --- a/src/backend/access/common/detoast.c +++ b/src/backend/access/common/detoast.c @@ -14,22 +14,17 @@ #include "postgres.h" #include "access/detoast.h" -#include "access/genam.h" -#include "access/heaptoast.h" #include "access/table.h" +#include "access/tableam.h" #include "access/toast_internals.h" #include "common/pg_lzcompress.h" #include "utils/expandeddatum.h" -#include "utils/fmgroids.h" #include "utils/rel.h" static struct varlena *toast_fetch_datum(struct varlena *attr); static struct varlena *toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength); -static void heap_fetch_toast_slice(Relation toastrel, Oid valueid, - int32 attrsize, int32 sliceoffset, - int32 slicelength, struct varlena *result); static struct varlena *toast_decompress_datum(struct varlena *attr); static struct varlena *toast_decompress_datum_slice(struct varlena *attr, int32 slicelength); @@ -356,8 +351,8 @@ toast_fetch_datum(struct varlena *attr) toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); /* Fetch all chunks */ - heap_fetch_toast_slice(toastrel, toast_pointer.va_valueid, attrsize, 0, - attrsize, result); + table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + attrsize, 0, attrsize, result); /* Close toast table */ table_close(toastrel, AccessShareLock); @@ -431,8 +426,9 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock); /* Fetch all chunks */ - heap_fetch_toast_slice(toastrel, toast_pointer.va_valueid, attrsize, - sliceoffset, slicelength, result); + table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid, + attrsize, sliceoffset, slicelength, + result); /* Close toast table */ table_close(toastrel, AccessShareLock); @@ -440,189 +436,6 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, return result; } -/* - * Fetch a TOAST slice from a heap table. - * - * toastrel is the relation from which chunks are to be fetched. - * valueid identifies the TOAST value from which chunks are being fetched. - * attrsize is the total size of the TOAST value. - * sliceoffset is the byte offset within the TOAST value from which to fetch. - * slicelength is the number of bytes to be fetched from the TOAST value. - * result is the varlena into which the results should be written. - */ -static void -heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, - int32 sliceoffset, int32 slicelength, - struct varlena *result) -{ - Relation *toastidxs; - ScanKeyData toastkey[3]; - TupleDesc toasttupDesc = toastrel->rd_att; - int nscankeys; - SysScanDesc toastscan; - HeapTuple ttup; - int32 expectedchunk; - int32 totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; - int startchunk; - int endchunk; - int num_indexes; - int validIndex; - SnapshotData SnapshotToast; - - /* Look for the valid index of toast relation */ - validIndex = toast_open_indexes(toastrel, - AccessShareLock, - &toastidxs, - &num_indexes); - - startchunk = sliceoffset / TOAST_MAX_CHUNK_SIZE; - endchunk = (sliceoffset + slicelength - 1) / TOAST_MAX_CHUNK_SIZE; - Assert(endchunk <= totalchunks); - - /* - * Setup a scan key to fetch from the index. This is either two keys or - * three depending on the number of chunks. - */ - ScanKeyInit(&toastkey[0], - (AttrNumber) 1, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(valueid)); - - /* - * No addition condition if fetching all chuns. Otherwise, use an - * equality condition for one chunk, and a range condition otherwise. - */ - if (startchunk == 0 && endchunk == totalchunks - 1) - nscankeys = 1; - else if (startchunk == endchunk) - { - ScanKeyInit(&toastkey[1], - (AttrNumber) 2, - BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(startchunk)); - nscankeys = 2; - } - else - { - ScanKeyInit(&toastkey[1], - (AttrNumber) 2, - BTGreaterEqualStrategyNumber, F_INT4GE, - Int32GetDatum(startchunk)); - ScanKeyInit(&toastkey[2], - (AttrNumber) 2, - BTLessEqualStrategyNumber, F_INT4LE, - Int32GetDatum(endchunk)); - nscankeys = 3; - } - - /* Prepare for scan */ - init_toast_snapshot(&SnapshotToast); - toastscan = systable_beginscan_ordered(toastrel, toastidxs[validIndex], - &SnapshotToast, nscankeys, toastkey); - - /* - * Read the chunks by index - * - * The index is on (valueid, chunkidx) so they will come in order - */ - expectedchunk = startchunk; - while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) - { - int32 curchunk; - Pointer chunk; - bool isnull; - char *chunkdata; - int32 chunksize; - int32 expected_size; - int32 chcpystrt; - int32 chcpyend; - - /* - * Have a chunk, extract the sequence number and the data - */ - curchunk = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull)); - Assert(!isnull); - chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); - Assert(!isnull); - if (!VARATT_IS_EXTENDED(chunk)) - { - chunksize = VARSIZE(chunk) - VARHDRSZ; - chunkdata = VARDATA(chunk); - } - else if (VARATT_IS_SHORT(chunk)) - { - /* could happen due to heap_form_tuple doing its thing */ - chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; - chunkdata = VARDATA_SHORT(chunk); - } - else - { - /* should never happen */ - elog(ERROR, "found toasted toast chunk for toast value %u in %s", - valueid, RelationGetRelationName(toastrel)); - chunksize = 0; /* keep compiler quiet */ - chunkdata = NULL; - } - - /* - * Some checks on the data we've found - */ - if (curchunk != expectedchunk) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (expected %d) for toast value %u in %s", - curchunk, expectedchunk, valueid, - RelationGetRelationName(toastrel)))); - if (curchunk > endchunk) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value %u in %s", - curchunk, - startchunk, endchunk, valueid, - RelationGetRelationName(toastrel)))); - expected_size = curchunk < totalchunks - 1 ? TOAST_MAX_CHUNK_SIZE - : attrsize - ((totalchunks - 1) * TOAST_MAX_CHUNK_SIZE); - if (chunksize != expected_size) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s", - chunksize, expected_size, - curchunk, totalchunks, valueid, - RelationGetRelationName(toastrel)))); - - /* - * Copy the data into proper place in our result - */ - chcpystrt = 0; - chcpyend = chunksize - 1; - if (curchunk == startchunk) - chcpystrt = sliceoffset % TOAST_MAX_CHUNK_SIZE; - if (curchunk == endchunk) - chcpyend = (sliceoffset + slicelength - 1) % TOAST_MAX_CHUNK_SIZE; - - memcpy(VARDATA(result) + - (curchunk * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt, - chunkdata + chcpystrt, - (chcpyend - chcpystrt) + 1); - - expectedchunk++; - } - - /* - * Final checks that we successfully fetched the datum - */ - if (expectedchunk != (endchunk + 1)) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg_internal("missing chunk number %d for toast value %u in %s", - expectedchunk, valueid, - RelationGetRelationName(toastrel)))); - - /* End scan and close indexes. */ - systable_endscan_ordered(toastscan); - toast_close_indexes(toastidxs, num_indexes, AccessShareLock); -} - /* ---------- * toast_decompress_datum - * diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 1d053e74e6..b443c6c1d5 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -2545,6 +2545,7 @@ static const TableAmRoutine heapam_methods = { .relation_size = table_block_relation_size, .relation_needs_toast_table = heapam_relation_needs_toast_table, .relation_toast_am = heapam_relation_toast_am, + .relation_fetch_toast_slice = heap_fetch_toast_slice, .relation_estimate_size = heapam_estimate_rel_size, diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index dcfdee4467..410920c1a7 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -25,10 +25,12 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/genam.h" #include "access/heapam.h" #include "access/heaptoast.h" #include "access/toast_helper.h" #include "access/toast_internals.h" +#include "utils/fmgroids.h" /* ---------- @@ -604,3 +606,186 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, return new_tuple; } + +/* + * Fetch a TOAST slice from a heap table. + * + * toastrel is the relation from which chunks are to be fetched. + * valueid identifies the TOAST value from which chunks are being fetched. + * attrsize is the total size of the TOAST value. + * sliceoffset is the byte offset within the TOAST value from which to fetch. + * slicelength is the number of bytes to be fetched from the TOAST value. + * result is the varlena into which the results should be written. + */ +void +heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, + int32 sliceoffset, int32 slicelength, + struct varlena *result) +{ + Relation *toastidxs; + ScanKeyData toastkey[3]; + TupleDesc toasttupDesc = toastrel->rd_att; + int nscankeys; + SysScanDesc toastscan; + HeapTuple ttup; + int32 expectedchunk; + int32 totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; + int startchunk; + int endchunk; + int num_indexes; + int validIndex; + SnapshotData SnapshotToast; + + /* Look for the valid index of toast relation */ + validIndex = toast_open_indexes(toastrel, + AccessShareLock, + &toastidxs, + &num_indexes); + + startchunk = sliceoffset / TOAST_MAX_CHUNK_SIZE; + endchunk = (sliceoffset + slicelength - 1) / TOAST_MAX_CHUNK_SIZE; + Assert(endchunk <= totalchunks); + + /* + * Setup a scan key to fetch from the index. This is either two keys or + * three depending on the number of chunks. + */ + ScanKeyInit(&toastkey[0], + (AttrNumber) 1, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(valueid)); + + /* + * No addition condition if fetching all chuns. Otherwise, use an + * equality condition for one chunk, and a range condition otherwise. + */ + if (startchunk == 0 && endchunk == totalchunks - 1) + nscankeys = 1; + else if (startchunk == endchunk) + { + ScanKeyInit(&toastkey[1], + (AttrNumber) 2, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(startchunk)); + nscankeys = 2; + } + else + { + ScanKeyInit(&toastkey[1], + (AttrNumber) 2, + BTGreaterEqualStrategyNumber, F_INT4GE, + Int32GetDatum(startchunk)); + ScanKeyInit(&toastkey[2], + (AttrNumber) 2, + BTLessEqualStrategyNumber, F_INT4LE, + Int32GetDatum(endchunk)); + nscankeys = 3; + } + + /* Prepare for scan */ + init_toast_snapshot(&SnapshotToast); + toastscan = systable_beginscan_ordered(toastrel, toastidxs[validIndex], + &SnapshotToast, nscankeys, toastkey); + + /* + * Read the chunks by index + * + * The index is on (valueid, chunkidx) so they will come in order + */ + expectedchunk = startchunk; + while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) + { + int32 curchunk; + Pointer chunk; + bool isnull; + char *chunkdata; + int32 chunksize; + int32 expected_size; + int32 chcpystrt; + int32 chcpyend; + + /* + * Have a chunk, extract the sequence number and the data + */ + curchunk = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull)); + Assert(!isnull); + chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); + Assert(!isnull); + if (!VARATT_IS_EXTENDED(chunk)) + { + chunksize = VARSIZE(chunk) - VARHDRSZ; + chunkdata = VARDATA(chunk); + } + else if (VARATT_IS_SHORT(chunk)) + { + /* could happen due to heap_form_tuple doing its thing */ + chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; + chunkdata = VARDATA_SHORT(chunk); + } + else + { + /* should never happen */ + elog(ERROR, "found toasted toast chunk for toast value %u in %s", + valueid, RelationGetRelationName(toastrel)); + chunksize = 0; /* keep compiler quiet */ + chunkdata = NULL; + } + + /* + * Some checks on the data we've found + */ + if (curchunk != expectedchunk) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("unexpected chunk number %d (expected %d) for toast value %u in %s", + curchunk, expectedchunk, valueid, + RelationGetRelationName(toastrel)))); + if (curchunk > endchunk) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast value %u in %s", + curchunk, + startchunk, endchunk, valueid, + RelationGetRelationName(toastrel)))); + expected_size = curchunk < totalchunks - 1 ? TOAST_MAX_CHUNK_SIZE + : attrsize - ((totalchunks - 1) * TOAST_MAX_CHUNK_SIZE); + if (chunksize != expected_size) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s", + chunksize, expected_size, + curchunk, totalchunks, valueid, + RelationGetRelationName(toastrel)))); + + /* + * Copy the data into proper place in our result + */ + chcpystrt = 0; + chcpyend = chunksize - 1; + if (curchunk == startchunk) + chcpystrt = sliceoffset % TOAST_MAX_CHUNK_SIZE; + if (curchunk == endchunk) + chcpyend = (sliceoffset + slicelength - 1) % TOAST_MAX_CHUNK_SIZE; + + memcpy(VARDATA(result) + + (curchunk * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt, + chunkdata + chcpystrt, + (chcpyend - chcpystrt) + 1); + + expectedchunk++; + } + + /* + * Final checks that we successfully fetched the datum + */ + if (expectedchunk != (endchunk + 1)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg_internal("missing chunk number %d for toast value %u in %s", + expectedchunk, valueid, + RelationGetRelationName(toastrel)))); + + /* End scan and close indexes. */ + systable_endscan_ordered(toastscan); + toast_close_indexes(toastidxs, num_indexes, AccessShareLock); +} diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 488a2e4a7f..0bf6b23c8f 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -136,4 +136,14 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull); +/* ---------- + * heap_fetch_toast_slice + * + * Fetch a slice from a toast value stored in a heap table. + * ---------- + */ +extern void heap_fetch_toast_slice(Relation toastrel, Oid valueid, + int32 attrsize, int32 sliceoffset, + int32 slicelength, struct varlena *result); + #endif /* HEAPTOAST_H */ diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 7ac9bb6d65..b784aaf5a9 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -588,6 +588,17 @@ typedef struct TableAmRoutine */ Oid (*relation_toast_am) (Relation rel); + /* + * This callback is invoked when detoasting a value stored in a toast + * table implemented by this AM. See table_relation_fetch_toast_slice() + * for more details. + */ + void (*relation_fetch_toast_slice) (Relation toastrel, Oid valueid, + int32 attrsize, + int32 sliceoffset, + int32 slicelength, + struct varlena *result); + /* ------------------------------------------------------------------------ * Planner related functions. @@ -1620,6 +1631,41 @@ table_relation_toast_am(Relation rel) return rel->rd_tableam->relation_toast_am(rel); } +/* + * Fetch all or part of a TOAST value from a TOAST table. + * + * If this AM is never used to implement a TOAST table, then this callback + * is not needed. But, if toasted values are ever stored in a table of this + * type, then you will need this callback. + * + * toastrel is the relation in which the toasted value is stored. + * + * valueid identifes which toast value is to be fetched. For the heap, + * this corresponds to the values stored in the chunk_id column. + * + * attrsize is the total size of the toast value to be fetched. + * + * sliceoffset is the offset within the toast value of the first byte that + * should be fetched. + * + * slicelength is the number of bytes from the toast value that should be + * fetched. + * + * result is caller-allocated space into which the fetched bytes should be + * stored. + */ +static inline void +table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, + int32 attrsize, int32 sliceoffset, + int32 slicelength, struct varlena *result) +{ + return toastrel->rd_tableam->relation_fetch_toast_slice(toastrel, valueid, + attrsize, + sliceoffset, + slicelength, + result); +} + /* ---------------------------------------------------------------------------- * Planner related functionality -- 2.17.2 (Apple Git-113)