Hi,
2008/10/1 Dimitri Fontaine <[EMAIL PROTECTED]>:
> -----BEGIN PGP SIGNED MESSAGE-----
> Hash: SHA1
>
> Hi,
>
> Le 30 sept. 08 à 20:03, Tom Lane a écrit :
>>
>> set_read_position(tupstore, &local_read_position);
>> tuple = tuplestore_gettuple(tupstore, ...);
>> get_read_position(tupstore, &local_read_position);
>>
>> rather than just tuplestore_gettuple. The set/get functions will be
>> cheap enough that this is no big deal. (Or maybe we should just
>> provide a wrapper function that does this sequence?)
>
> It seems to me to share some ideas with the MemoryContext concept: what
> about a TupstoreContext associated with tuplestore, you get a common default
> one if you don't register your own, and use
> tuplestore_gettuple(MyTupstoreContext, ...);
>
> Maybe some other API would benefit from the idea?
>
I'm just working on tuplestore recording multiple positions for my
window function project. Attached patch is still in progress but seems
it works in a situation.
From my work, the setting/getting read position and delegate savig
positions to the caller will probably have problems, because of memory
control for saving positions and tuplestore status changing (memory ->
BufFile). Instead, I decided it'd better that we can indicate the row
number by integer.
Regards,
--
Hitoshi Harada
*** a/src/backend/utils/sort/tuplestore.c
--- b/src/backend/utils/sort/tuplestore.c
***************
*** 141,148 **** struct Tuplestorestate
--- 141,159 ----
int markpos_current; /* saved "current" */
int markpos_file; /* saved "readpos_file" */
off_t markpos_offset; /* saved "readpos_offset" */
+
+ /*
+ * When asked EXEC_FLAG_SETPOS, we need record multiple state->myfile pointer.
+ * This pointer is used to optimize offset calculation closer to O(1).
+ * npos_file is fixed layout file since only int + off_t will be needed to restore
+ * tuple positions. int + off_t may get better by struct{int, off_t} to reduce call
+ * BufFileWrite and BufFileRead?
+ * This feature is off unless eflags is set.
+ */
+ BufFile *npos_file;
};
+
#define COPYTUP(state,tup) ((*(state)->copytup) (state, tup))
#define WRITETUP(state,tup) ((*(state)->writetup) (state, tup))
#define READTUP(state,len) ((*(state)->readtup) (state, len))
***************
*** 150,155 **** struct Tuplestorestate
--- 161,175 ----
#define USEMEM(state,amt) ((state)->availMem -= (amt))
#define FREEMEM(state,amt) ((state)->availMem += (amt))
+ #define RecordFilePointer(state,file,offset) do{\
+ BufFileTell((state)->myfile, \
+ &(file), &(offset)); \
+ BufFileWrite((state)->npos_file, \
+ &(file), sizeof(int)); \
+ BufFileWrite((state)->npos_file, \
+ &(offset), sizeof(off_t)); \
+ }while(0)
+
/*--------------------
*
* NOTES about on-tape representation of tuples:
***************
*** 200,205 **** static Tuplestorestate *tuplestore_begin_common(int eflags,
--- 220,226 ----
int maxKBytes);
static void tuplestore_puttuple_common(Tuplestorestate *state, void *tuple);
static void dumptuples(Tuplestorestate *state);
+ static void tuplestore_recordpos(Tuplestorestate *state);
static void tuplestore_trim(Tuplestorestate *state, int ntuples);
static unsigned int getlen(Tuplestorestate *state, bool eofOK);
static void *copytup_heap(Tuplestorestate *state, void *tup);
***************
*** 234,239 **** tuplestore_begin_common(int eflags, bool interXact, int maxKBytes)
--- 255,262 ----
state->eof_reached = false;
state->current = 0;
+ state->npos_file = NULL;
+
return state;
}
***************
*** 291,296 **** tuplestore_begin_heap(bool randomAccess, bool interXact, int maxKBytes)
--- 314,320 ----
* EXEC_FLAG_REWIND need rewind to start
* EXEC_FLAG_BACKWARD need backward fetch
* EXEC_FLAG_MARK need mark/restore
+ * EXEC_FLAG_SETPOS need setpos
* If tuplestore_set_eflags is not called, REWIND and MARK are allowed,
* and BACKWARD is set per "randomAccess" in the tuplestore_begin_xxx call.
*/
***************
*** 315,320 **** tuplestore_end(Tuplestorestate *state)
--- 339,346 ----
if (state->myfile)
BufFileClose(state->myfile);
+ if (state->npos_file)
+ BufFileClose(state->npos_file);
if (state->memtuples)
{
for (i = 0; i < state->memtupcount; i++)
***************
*** 443,452 **** tuplestore_puttuple_common(Tuplestorestate *state, void *tuple)
--- 469,481 ----
*/
PrepareTempTablespaces();
state->myfile = BufFileCreateTemp(state->interXact);
+ if (state->eflags & EXEC_FLAG_SETPOS)
+ state->npos_file = BufFileCreateTemp(state->interXact);
state->status = TSS_WRITEFILE;
dumptuples(state);
break;
case TSS_WRITEFILE:
+ tuplestore_recordpos(state);
WRITETUP(state, tuple);
break;
case TSS_READFILE:
***************
*** 461,466 **** tuplestore_puttuple_common(Tuplestorestate *state, void *tuple)
--- 490,496 ----
state->writepos_file, state->writepos_offset,
SEEK_SET) != 0)
elog(ERROR, "seek to EOF failed");
+ tuplestore_recordpos(state);
state->status = TSS_WRITEFILE;
WRITETUP(state, tuple);
break;
***************
*** 678,683 **** static void
--- 708,715 ----
dumptuples(Tuplestorestate *state)
{
int i;
+ int file;
+ off_t offset;
for (i = 0;; i++)
{
***************
*** 687,692 **** dumptuples(Tuplestorestate *state)
--- 719,730 ----
if (i == state->markpos_current)
BufFileTell(state->myfile,
&state->markpos_file, &state->markpos_offset);
+
+ if (state->npos_file)
+ {
+ RecordFilePointer(state, file, offset);
+ }
+
if (i >= state->memtupcount)
break;
WRITETUP(state, state->memtuples[i]);
***************
*** 812,817 **** tuplestore_restorepos(Tuplestorestate *state)
--- 850,931 ----
}
/*
+ * tuplestore_setpos -
+ */
+ void
+ tuplestore_setpos(Tuplestorestate *state, int nth)
+ {
+ int file, file_npos;
+ off_t offset, offset_npos;
+
+ Assert((state->eflags & EXEC_FLAG_SETPOS) && (0 <= nth));
+
+ switch (state->status)
+ {
+ case TSS_INMEM:
+ state->eof_reached = false;
+ state->current = nth;
+ break;
+ case TSS_WRITEFILE:
+ case TSS_READFILE:
+ /*
+ * It is quite redundant and may have to be optimized not to
+ * read buffer back and force. But we don't know if we call
+ * setpos so frequently.
+ */
+ state->eof_reached = false;
+ BufFileTell(state->npos_file,
+ &file_npos, &offset_npos);
+ if (!(
+ (BufFileSeekBlock(state->npos_file,
+ nth * (sizeof(int) + sizeof(off_t)) / BLCKSZ) == 0) &&
+ (BufFileSeek(state->npos_file, 0,
+ nth * (sizeof(int) + sizeof(off_t)) % BLCKSZ,
+ SEEK_CUR) == 0) &&
+ (BufFileRead(state->npos_file,
+ &file, sizeof(int)) == 0) &&
+ (BufFileRead(state->npos_file,
+ &offset, sizeof(off_t)) == 0) &&
+ (BufFileSeek(state->npos_file,
+ file_npos, offset_npos, SEEK_SET) == 0)))
+ elog(ERROR, "tuplestore_setpos failed while seeking for tuple position");
+
+ if (state->status == TSS_WRITEFILE)
+ {
+ state->readpos_file = file;
+ state->readpos_offset = offset;
+ }
+ else
+ {
+ if (BufFileSeek(state->myfile,
+ file, offset, SEEK_SET) != 0)
+ elog(ERROR, "tuplestore_setpos failed");
+ }
+ break;
+ default:
+ elog(ERROR, "invalid tuplestore state");
+ break;
+ }
+ }
+
+ /*
+ *
+ */
+ static void
+ tuplestore_recordpos(Tuplestorestate *state)
+ {
+ int file;
+ off_t offset;
+
+ Assert(state->status != TSS_INMEM);
+
+ if (!state->npos_file)
+ return;
+
+ RecordFilePointer(state, file, offset);
+ }
+
+ /*
* tuplestore_trim - remove all but ntuples tuples before current
*/
static void
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
