Zdenek Kotala wrote:
> Alvaro Herrera napsal(a):
>> Heikki Linnakangas wrote:
>>
>>> Hmm, you're right. I think it can be made to work by storing the
>>> *end* offset of each chunk. To find the chunk containing offset X,
>>> search for the first chunk with end_offset > X.
>>
>> FWIW I'm trying to do this. So far I've managed to make the basic thing
>> work, and I'm about to have a look at the slice interface.
Okay, so this seems to work. It's missing writing the sanity checks on
the returned data, and a look at the SGML docs to see if anything needs
updating. I'm also going to recheck code comments that may need
updates.
--
Alvaro Herrera http://www.CommandPrompt.com/
PostgreSQL Replication, Consulting, Custom Development, 24x7 support
Index: src/backend/access/heap/tuptoaster.c
===================================================================
RCS file: /home/alvherre/Code/cvs/pgsql/src/backend/access/heap/tuptoaster.c,v
retrieving revision 1.91
diff -c -p -r1.91 tuptoaster.c
*** src/backend/access/heap/tuptoaster.c 6 Nov 2008 20:51:14 -0000 1.91
--- src/backend/access/heap/tuptoaster.c 17 Nov 2008 20:49:41 -0000
*************** toast_save_datum(Relation rel, Datum val
*** 1134,1140 ****
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
! int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
--- 1134,1140 ----
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
! int32 data_done = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
*************** toast_save_datum(Relation rel, Datum val
*** 1208,1214 ****
/*
* Build a tuple and store it
*/
! t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
--- 1208,1214 ----
/*
* Build a tuple and store it
*/
! t_values[1] = Int32GetDatum(data_done + chunk_size);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
*************** toast_save_datum(Relation rel, Datum val
*** 1237,1242 ****
--- 1237,1243 ----
*/
data_todo -= chunk_size;
data_p += chunk_size;
+ data_done += chunk_size;
}
/*
*************** toast_fetch_datum(struct varlena * attr)
*** 1336,1343 ****
struct varlena *result;
struct varatt_external toast_pointer;
int32 ressize;
! int32 residx,
! nextidx;
int32 numchunks;
Pointer chunk;
bool isnull;
--- 1337,1344 ----
struct varlena *result;
struct varatt_external toast_pointer;
int32 ressize;
! int32 endoff,
! prevend;
int32 numchunks;
Pointer chunk;
bool isnull;
*************** toast_fetch_datum(struct varlena * attr)
*** 1373,1394 ****
ObjectIdGetDatum(toast_pointer.va_valueid));
/*
! * Read the chunks by index
*
! * Note that because the index is actually on (valueid, chunkidx) we will
! * see the chunks in chunkidx order, even though we didn't explicitly ask
* for it.
*/
! nextidx = 0;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
SnapshotToast, 1, &toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
/*
! * Have a chunk, extract the sequence number and the data
*/
! residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
--- 1374,1395 ----
ObjectIdGetDatum(toast_pointer.va_valueid));
/*
! * Read the chunks by chuck end position
*
! * Note that because the index is actually on (valueid, chunk-end) we will
! * see the chunks in chunk-end order, even though we didn't explicitly ask
* for it.
*/
! prevend = 0;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
SnapshotToast, 1, &toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
/*
! * Have a chunk, extract its end offset and the data
*/
! endoff = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
*************** toast_fetch_datum(struct varlena * attr)
*** 1416,1472 ****
/*
* Some checks on the data we've found
*/
! if (residx != nextidx)
! elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u in %s",
! residx, nextidx,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! if (residx < numchunks - 1)
! {
! if (chunksize != TOAST_MAX_CHUNK_SIZE)
! elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s",
! chunksize, (int) TOAST_MAX_CHUNK_SIZE,
! residx, numchunks,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! }
! else if (residx == numchunks - 1)
! {
! if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize)
! elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u in %s",
! chunksize,
! (int) (ressize - residx * TOAST_MAX_CHUNK_SIZE),
! residx,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! }
! else
! elog(ERROR, "unexpected chunk number %d (out of range %d..%d) for toast value %u in %s",
! residx,
! 0, numchunks - 1,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* Copy the data into proper place in our result
*/
! memcpy(VARDATA(result) + residx * TOAST_MAX_CHUNK_SIZE,
chunkdata,
chunksize);
! nextidx++;
}
/*
- * Final checks that we successfully fetched the datum
- */
- if (nextidx != numchunks)
- elog(ERROR, "missing chunk number %d for toast value %u in %s",
- nextidx,
- toast_pointer.va_valueid,
- RelationGetRelationName(toastrel));
-
- /*
* End scan and close relations
*/
systable_endscan_ordered(toastscan);
--- 1417,1439 ----
/*
* Some checks on the data we've found
*/
! if (endoff <= prevend)
! elog(ERROR, "unexpected chunk end position %d (expected > %d) for toast value %u in %s",
! endoff, prevend,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* Copy the data into proper place in our result
*/
! memcpy(VARDATA(result) + prevend,
chunkdata,
chunksize);
! prevend = endoff;
}
/*
* End scan and close relations
*/
systable_endscan_ordered(toastscan);
*************** toast_fetch_datum_slice(struct varlena *
*** 1488,1515 ****
{
Relation toastrel;
Relation toastidx;
! ScanKeyData toastkey[3];
! int nscankeys;
SysScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
struct varlena *result;
struct varatt_external toast_pointer;
int32 attrsize;
! int32 residx;
! int32 nextidx;
! int numchunks;
! int startchunk;
! int endchunk;
! int32 startoffset;
! int32 endoffset;
! int totalchunks;
! Pointer chunk;
! bool isnull;
! char *chunkdata;
! int32 chunksize;
! int32 chcpystrt;
! int32 chcpyend;
Assert(VARATT_IS_EXTERNAL(attr));
--- 1455,1468 ----
{
Relation toastrel;
Relation toastidx;
! ScanKeyData toastkey[2];
SysScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
struct varlena *result;
struct varatt_external toast_pointer;
int32 attrsize;
! int32 dstoffset;
Assert(VARATT_IS_EXTERNAL(attr));
*************** toast_fetch_datum_slice(struct varlena *
*** 1523,1529 ****
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
attrsize = toast_pointer.va_extsize;
- totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
if (sliceoffset >= attrsize)
{
--- 1476,1481 ----
*************** toast_fetch_datum_slice(struct varlena *
*** 1544,1610 ****
if (length == 0)
return result; /* Can save a lot of work at this point! */
- startchunk = sliceoffset / TOAST_MAX_CHUNK_SIZE;
- endchunk = (sliceoffset + length - 1) / TOAST_MAX_CHUNK_SIZE;
- numchunks = (endchunk - startchunk) + 1;
-
- startoffset = sliceoffset % TOAST_MAX_CHUNK_SIZE;
- endoffset = (sliceoffset + length - 1) % TOAST_MAX_CHUNK_SIZE;
-
/*
* Open the toast relation and its index
*/
toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock);
! toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
! * Setup a scan key to fetch from the index. This is either two keys or
! * three depending on the number of chunks.
*/
ScanKeyInit(&toastkey[0],
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(toast_pointer.va_valueid));
/*
! * Use equality condition for one chunk, a range condition otherwise:
! */
! if (numchunks == 1)
! {
! ScanKeyInit(&toastkey[1],
! (AttrNumber) 2,
! BTEqualStrategyNumber, F_INT4EQ,
! Int32GetDatum(startchunk));
! nscankeys = 2;
! }
! else
! {
! ScanKeyInit(&toastkey[1],
! (AttrNumber) 2,
! BTGreaterEqualStrategyNumber, F_INT4GE,
! Int32GetDatum(startchunk));
! ScanKeyInit(&toastkey[2],
! (AttrNumber) 2,
! BTLessEqualStrategyNumber, F_INT4LE,
! Int32GetDatum(endchunk));
! nscankeys = 3;
! }
!
! /*
! * Read the chunks by index
*
! * The index is on (valueid, chunkidx) so they will come in order
*/
! nextidx = startchunk;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
! SnapshotToast, nscankeys, toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
/*
! * Have a chunk, extract the sequence number and the data
*/
! residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
--- 1496,1542 ----
if (length == 0)
return result; /* Can save a lot of work at this point! */
/*
* Open the toast relation and its index
*/
toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock);
! toasttupDesc = RelationGetDescr(toastrel);
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
! * Setup a scan key to fetch from the index.
*/
ScanKeyInit(&toastkey[0],
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(toast_pointer.va_valueid));
+ ScanKeyInit(&toastkey[1],
+ (AttrNumber) 2,
+ BTGreaterStrategyNumber, F_INT4GT,
+ Int32GetDatum(sliceoffset));
/*
! * Read the chunks by end offset
*
! * The index is on (valueid, chunk-end) so they will come in order
*/
! dstoffset = 0;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
! SnapshotToast, 2, toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
+ uint32 srcstart;
+ uint32 chunkend;
+ uint32 copylength;
+ Pointer chunk;
+ bool isnull;
+ char *chunkdata;
+ int32 chunksize;
+
/*
! * Have a chunk, extract the end offset and the data
*/
! chunkend = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
*************** toast_fetch_datum_slice(struct varlena *
*** 1629,1694 ****
chunkdata = NULL;
}
/*
* Some checks on the data we've found
*/
! if ((residx != nextidx) || (residx > endchunk) || (residx < startchunk))
! elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u in %s",
! residx, nextidx,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! if (residx < totalchunks - 1)
! {
! if (chunksize != TOAST_MAX_CHUNK_SIZE)
! elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s when fetching slice",
! chunksize, (int) TOAST_MAX_CHUNK_SIZE,
! residx, totalchunks,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! }
! else if (residx == totalchunks - 1)
! {
! if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != attrsize)
! elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u in %s when fetching slice",
! chunksize,
! (int) (attrsize - residx * TOAST_MAX_CHUNK_SIZE),
! residx,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
! }
! else
! elog(ERROR, "unexpected chunk number %d (out of range %d..%d) for toast value %u in %s",
! residx,
! 0, totalchunks - 1,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
/*
* Copy the data into proper place in our result
*/
! chcpystrt = 0;
! chcpyend = chunksize - 1;
! if (residx == startchunk)
! chcpystrt = startoffset;
! if (residx == endchunk)
! chcpyend = endoffset;
!
! memcpy(VARDATA(result) +
! (residx * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt,
! chunkdata + chcpystrt,
! (chcpyend - chcpystrt) + 1);
! nextidx++;
}
/*
* Final checks that we successfully fetched the datum
*/
! if (nextidx != (endchunk + 1))
! elog(ERROR, "missing chunk number %d for toast value %u in %s",
! nextidx,
! toast_pointer.va_valueid,
! RelationGetRelationName(toastrel));
/*
* End scan and close relations
--- 1561,1598 ----
chunkdata = NULL;
}
+ #if 0
/*
* Some checks on the data we've found
*/
! #endif
/*
* Copy the data into proper place in our result
*/
! if (dstoffset == 0) /* first chunk; skip unneeded bytes */
! srcstart = sliceoffset - chunkend + chunksize;
! else
! srcstart = 0;
!
! copylength = Min(chunksize - srcstart, length);
!
! memcpy(VARDATA(result) + dstoffset,
! chunkdata + srcstart,
! copylength);
!
! length -= copylength;
! dstoffset += copylength;
! if (length == 0)
! break;
}
+ #if 0
/*
* Final checks that we successfully fetched the datum
*/
! #endif
/*
* End scan and close relations
Index: src/backend/catalog/toasting.c
===================================================================
RCS file: /home/alvherre/Code/cvs/pgsql/src/backend/catalog/toasting.c,v
retrieving revision 1.11
diff -c -p -r1.11 toasting.c
*** src/backend/catalog/toasting.c 1 Sep 2008 20:42:43 -0000 1.11
--- src/backend/catalog/toasting.c 14 Nov 2008 23:42:02 -0000
*************** create_toast_table(Relation rel, Oid toa
*** 157,163 ****
OIDOID,
-1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 2,
! "chunk_seq",
INT4OID,
-1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 3,
--- 157,163 ----
OIDOID,
-1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 2,
! "chunk_end",
INT4OID,
-1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 3,
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
