Changeset: 1309e4680b0d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1309e4680b0d
Modified Files:
clients/mapiclient/mclient.c
gdk/gdk_utils.mx
monetdb5/optimizer/opt_mitosis.mx
sql/storage/bat/bat_storage.c
testing/Mtest.py.in
Branch: Dec2011
Log Message:
Mtest.py: prevent "IndexError: list index out of range"
diffs (152 lines):
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -966,6 +966,7 @@ TESTrenderer(MapiHdl hdl)
}
mnstr_printf(toConsole, "\t]\n");
}
+ fprintf(stderr, "# Timer %7ld.%03ld msec \n", (long) ((th - t0) /
1000), (long) ((th - t0) % 1000));
}
static void
@@ -1431,9 +1432,11 @@ format_result(Mapi mid, MapiHdl hdl, cha
case Q_UPDATE:
SQLqueryEcho(hdl);
if (formatter == RAWformatter ||
- formatter == TESTformatter)
+ formatter == TESTformatter) {
+ timerHumanStop();
mnstr_printf(toConsole, "[ " LLFMT "\t]\n",
mapi_rows_affected(hdl));
- else {
+ fprintf(stderr, "# Timer %7ld.%03ld msec \n",
(long) ((th - t0) / 1000), (long) ((th - t0) % 1000));
+ } else {
timerHumanStop();
aff = mapi_rows_affected(hdl);
lid = mapi_get_last_id(hdl);
diff --git a/gdk/gdk_utils.mx b/gdk/gdk_utils.mx
--- a/gdk/gdk_utils.mx
+++ b/gdk/gdk_utils.mx
@@ -1674,8 +1674,12 @@ GDKinit(opt *set, int setlen)
GDKlockHome();
/* Mserver by default takes 80% of all memory as a default */
- GDK_mem_maxsize_max = (size_t) ((double) MT_npages() * (double)
MT_pagesize() * 0.815);
- GDK_mmap_minsize = GDK_mem_maxsize = GDK_mem_maxsize_max;
+ GDK_mem_maxsize = GDK_mem_maxsize_max = (size_t) ((double) MT_npages()
* (double) MT_pagesize() * 0.815);
+#ifdef NATIVE_WIN32
+ GDK_mmap_minsize = GDK_mem_maxsize_max;
+#else
+ GDK_mmap_minsize = GDK_mem_bigsize = MIN( 1<<30 , GDK_mem_maxsize_max );
+#endif
GDKmemchk(TRUE, TRUE);
GDKremovedir(DELDIR);
BBPinit();
@@ -1746,10 +1750,9 @@ GDKinit(opt *set, int setlen)
if (GDKnr_threads == 0)
GDKnr_threads = MT_check_nr_cores();
#ifdef NATIVE_WIN32
- if (GDKnr_threads)
- GDK_mmap_minsize /= (GDKnr_threads? GDKnr_threads: 1);
+ GDK_mmap_minsize /= (GDKnr_threads ? GDKnr_threads : 1);
#else
- GDK_mmap_minsize = 256 * 1024 * 1024;
+ GDK_mmap_minsize = GDK_mem_bigsize = MIN( 1<<30 , GDK_mem_maxsize_max /
(GDKnr_threads ? GDKnr_threads : 1) );
#endif
#ifdef HAVE_POSIX_FADVISE
diff --git a/monetdb5/optimizer/opt_mitosis.mx
b/monetdb5/optimizer/opt_mitosis.mx
--- a/monetdb5/optimizer/opt_mitosis.mx
+++ b/monetdb5/optimizer/opt_mitosis.mx
@@ -63,7 +63,7 @@ comment "Modify the plan to exploit para
#include "opt_support.h"
#define MAXSLICES 256 /* to be refined */
-#define MINPARTCNT 100000 /* minimal record count per partition */
+#define MINPARTCNT 1000000 /* minimal record count per partition */
@:exportOptimizer(mitosis)@
#define OPTDEBUGmitosis if ( optDebug & ((lng)1 <<DEBUG_OPT_MITOSIS) )
#endif
@@ -73,8 +73,6 @@ comment "Modify the plan to exploit para
#include "opt_octopus.h"
#include "mal_interpreter.h"
-#define PARTITION_THRESHOLD (wrd) threads /* should be increased in
production version */
-
static int eligible(MalBlkPtr mb )
{
InstrPtr p;
@@ -97,10 +95,10 @@ OPTmitosisImplementation(Client cntxt, M
int i, j, k, limit, estimate=0, tpe, pieces=1;
str schema=0, table=0;
VarRecord low,hgh;
- oid slice;
+ BUN slice;
wrd r = 0, rowcnt=0; /* table should be sizeable to consider
parallel execution*/
InstrPtr q,*old, target= 0, matq;
- size_t argsize= 3 *sizeof(lng); /* 2 arguments and a result */
+ size_t argsize= 3 *2 /* or 4? */ *sizeof(lng); /* 2 arguments and a
result */
int threads = GDKnr_threads ? GDKnr_threads:1;
ValRecord vr;
VarPtr loc,rows;
@@ -142,22 +140,30 @@ OPTmitosisImplementation(Client cntxt, M
* For the time being we just go for pieces that fit into memory in
isolation.
* A fictive rowcount is derived based on argument types, such that all
pieces
* would fit into memory conveniently for processing. We attempt to use
- * not more threads then strictly needed.
+ * not more threads than strictly needed.
* Experience shows that the pieces should not be too small.
* If we should limit to |threads| is still an open issue.
*/
if ( (i = OPTlegAdviceInternal(mb,stk,p)) > 0 )
pieces = i;
else {
- /* ensure that GDKnr_threads partitions fit into main memory */
- r = (BUN) (monet_memory / argsize / 4);
- if ( (r? (int) rowcnt/r:1) < (wrd) threads && r > MINPARTCNT)
- pieces = (r? (int) rowcnt/r:1);
+ r = (wrd) (monet_memory / argsize);
+ /* if data exceeds memory size,
+ * i.e., (rowcnt*argsize > monet_memory),
+ * i.e., (rowcnt > monet_memory/argsize = r) */
+ if (rowcnt > r && r/threads > 0)
+ /* create |pieces| > |threads| partitions such that
+ * |threads| partitions at a time fit in memory,
+ * i.e., (threads*(rowcnt/pieces) <= r),
+ * i.e., (rowcnt/pieces <= r/threads),
+ * i.e., (pieces => rowcnt/(r/threads))
+ * (assuming that (r > threads*MINPARTCNT)) */
+ pieces = (int) (rowcnt / (r/threads)) + 1;
else
/* exploit parallelism, but ensure minimal partition size to
limit overhead */
if (rowcnt > MINPARTCNT)
pieces = (int) MIN ( (rowcnt / MINPARTCNT) , (wrd)
threads);
- /* when testing, split also small BATs, but avoid empty pieces
*/
+ /* when testing, always aim for full parallelism, but avoid
empty pieces */
FORCEMITODEBUG
if (pieces < threads)
pieces = (int) MIN ( (wrd) threads , rowcnt );
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -250,7 +250,8 @@ delta_append_bat( sql_delta *bat, BAT *i
/* We simply use the to be inserted bat directly.
* Disabled this optimization: sometimes the bat is used later in the
* mal plan.
- * This should be solved by changing the input into a view (somehow)
+ * This should be solved by changing the input into a view (somehow).
+ * Alternatively, COPY INTO ... LOCKED can/should be used.
if (BATcount(b) == 0 && !isVIEW(i) && BBP_lrefs(i->batCacheid) <= 1 &&
i->htype == TYPE_void && i->ttype != TYPE_void && bat->ibase == i->H->seq){
temp_destroy(bat->ibid);
bat->ibid = temp_create(i);
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -1869,7 +1869,7 @@ def RunTest(env, TST, BusyPorts, COND, o
if corefile is None:
corefile = os.path.join(env['GDK_DBFARM'], env['TSTDB'],
'core.*')
corefile = glob.glob(corefile)
- if corefile is not None and os.path.exists(corefile[0]):
+ if corefile is not None and len(corefile) > 0 and
os.path.exists(corefile[0]):
corefile = corefile[0]
else:
corefile = None
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
