MonetDB: default - Capture some system properties
Changeset: 5f574090b78c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5f574090b78c
Modified Files:
clients/Tests/exports.stable.out
clients/mapiclient/eventparser.c
clients/mapiclient/eventparser.h
clients/mapiclient/tachograph.c
monetdb5/mal/mal.c
monetdb5/mal/mal.h
monetdb5/mal/mal_profiler.c
tools/mserver/mserver5.c
Branch: default
Log Message:
Capture some system properties
Keep the monet_hello message around and sent it for consumption
to the profilers. This could ease off-line analysis of the
performance.
diffs (206 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2319,6 +2319,7 @@ int isLoaded(str modulename);
int isLoopBarrier(MalBlkPtr mb, int pc);
int isMapOp(InstrPtr q);
int isMatJoinOp(InstrPtr q);
+int isMatLeftJoinOp(InstrPtr q);
int isModuleDefined(Module scope, str name);
int isMultiplex(InstrPtr q);
int isNotUsedIn(InstrPtr p, int start, int a);
@@ -2446,6 +2447,7 @@ int mnstr_writeInt_wrap(Stream *S, int *
str mnstr_writeIntwrap(void *ret, Stream *S, int *data);
int mnstr_write_string(Stream *S, str data);
str mnstr_write_stringwrap(void *ret, Stream *S, str *data);
+char monet_characteristics[PATHLENGTH];
char monet_cwd[PATHLENGTH];
size_t monet_memory;
void moveInstruction(MalBlkPtr mb, int pc, int target);
@@ -2681,6 +2683,7 @@ str subdeltaRef;
str subgroupRef;
str subgroupdoneRef;
str subjoinRef;
+str subleftjoinRef;
str submaxRef;
str submedianRef;
str subminRef;
diff --git a/clients/mapiclient/eventparser.c b/clients/mapiclient/eventparser.c
--- a/clients/mapiclient/eventparser.c
+++ b/clients/mapiclient/eventparser.c
@@ -17,6 +17,7 @@ int malargtop;
char *malvariables[MAXMALARGS];
int malvartop;
int debug;
+char *monet_characteristics;
void
clearArguments(void)
@@ -105,8 +106,11 @@ eventparser(char *row, EventRecord *ev)
struct tm stm;
/* check basic validaty first */
- if (row[0] =='#')
+ if (row[0] =='#'){
+ if( row[1] =='{')
+ monet_characteristics = strdup(row+1);
return 0;
+ }
if (row[0] != '[')
return -1;
if ((cc= strrchr(row,']')) == 0 || *(cc+1) !=0)
diff --git a/clients/mapiclient/eventparser.h b/clients/mapiclient/eventparser.h
--- a/clients/mapiclient/eventparser.h
+++ b/clients/mapiclient/eventparser.h
@@ -93,6 +93,7 @@ extern int malargtop;
extern char *malvariables[MAXMALARGS];
extern int malvartop;
extern int debug;
+extern char *monet_characteristics;
extern void clearArguments(void);
extern int eventparser(char *row, EventRecord *ev);
diff --git a/clients/mapiclient/tachograph.c b/clients/mapiclient/tachograph.c
--- a/clients/mapiclient/tachograph.c
+++ b/clients/mapiclient/tachograph.c
@@ -64,7 +64,7 @@
static stream *conn = NULL;
static char hostname[128];
-static char *basefilename = tacho;
+static char *basefilename = tachograph;
static char *cache= cache;
static char *dbname;
static int beat = 5000;
@@ -478,6 +478,8 @@ initFiles(void)
fprintf(stderr,Could not create %s\n,buf);
exit(0);
}
+ if( monet_characteristics)
+ fprintf(tachotrace,%s\n,monet_characteristics);
}
static void
diff --git a/monetdb5/mal/mal.c b/monetdb5/mal/mal.c
--- a/monetdb5/mal/mal.c
+++ b/monetdb5/mal/mal.c
@@ -12,6 +12,7 @@
char monet_cwd[PATHLENGTH] = { 0 };
size_t monet_memory;
+char monet_characteristics[PATHLENGTH];
char *mal_trace; /* enable profile events on console */
#include mal_stack.h
diff --git a/monetdb5/mal/mal.h b/monetdb5/mal/mal.h
--- a/monetdb5/mal/mal.h
+++ b/monetdb5/mal/mal.h
@@ -42,6 +42,7 @@
mal_export char monet_cwd[PATHLENGTH];
mal_export size_t monet_memory;
+mal_export charmonet_characteristics[PATHLENGTH];
mal_export lng memorypool; /* memory claimed by
concurrent threads */
mal_export int memoryclaims;/* number of threads active
with expensive operations */
mal_export char*mal_trace; /* enable profile
events on console */
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -55,9 +55,11 @@ offlineProfilerHeader(void)
char logbuffer[LOGLEN], *logbase;
int loglen;
- if (eventstream == NULL) {
+ if (eventstream == NULL)
return ;
- }
+ mnstr_printf(eventstream,%s\n, monet_characteristics);
+ mnstr_flush(eventstream);
+
lognew();
logadd(# );
logadd(event,\t);
@@ -81,10 +83,8 @@ offlineProfilerHeader(void)
#endif
logadd(stmt,\t);
logadd(# name \n);
- if (eventstream){
- mnstr_printf(eventstream,%s\n,
MonetDB: default - Collect traces in cache
Changeset: 774663a7a97d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=774663a7a97d
Modified Files:
clients/mapiclient/tachograph.c
Branch: default
Log Message:
Collect traces in cache
The individual query traces are collected in separate files
in a local cache. The number of traces is compile time limited.
These traces can be used for e.g.tomograph analysis.
Note: still assuming non-concurrent query stream
diffs (292 lines):
diff --git a/clients/mapiclient/tachograph.c b/clients/mapiclient/tachograph.c
--- a/clients/mapiclient/tachograph.c
+++ b/clients/mapiclient/tachograph.c
@@ -6,7 +6,7 @@
* Copyright 2008-2015 MonetDB B.V.
*/
-/* (c) M Kersten
+/* author: M Kersten
* Progress indicator
* tachograph -d demo
* which connects to the demo database server and presents a server progress
bar.
@@ -65,6 +65,7 @@
static stream *conn = NULL;
static char hostname[128];
static char *basefilename = tacho;
+static char *cache= cache;
static char *dbname;
static int beat = 5000;
static int delay = 0; // ms
@@ -194,7 +195,12 @@ int prevprogress =0;
int prevlevel =0;
size_t txtlength=0;
-static FILE *tachofd;
+// limit the number of separate queries in the pool
+#define QUERYPOOL 32
+int queryid= 0;
+
+static FILE *tachojson;
+static FILE *tachotrace;
static void resetTachograph(void){
int i;
@@ -218,8 +224,10 @@ static void resetTachograph(void){
starttime = 0;
finishtime = 0;
duration =0;
- fclose(tachofd);
- tachofd = 0;
+ fclose(tachojson);
+ tachojson = 0;
+ fclose(tachotrace);
+ tachotrace = 0;
prevprogress = 0;
txtlength =0;
prevlevel=0;
@@ -227,6 +235,7 @@ static void resetTachograph(void){
pccount = 0;
fflush(stdout);
events = 0;
+ queryid = (queryid+1) % QUERYPOOL;
}
static char stamp[BUFSIZ]={0};
@@ -434,27 +443,55 @@ showBar(int level, lng clk, char *stmt)
prevlevel = level;
}
-/* create the progressbar JSON file for pickup */
+/* create the progressbar JSON file for pickup.
+ * Keep the file in the pool, together with its original trace
+ */
+
+static void
+initFiles(void)
+{
+ char buf[BUFSIZ];
+
+ if (cache)
+#ifdef NATIVE_WIN32
+
snprintf(buf,BUFSIZ,%s\\%s_%s_%d.json,cache,basefilename,dbname, queryid);
+#else
+
snprintf(buf,BUFSIZ,%s/%s_%s_%d.json,cache,basefilename,dbname, queryid);
+#endif
+ else
+ snprintf(buf,BUFSIZ,%s_%s_%d.json,basefilename,dbname,
queryid);
+ tachojson= fopen(buf,w);
+ if( tachojson == NULL){
+ fprintf(stderr,Could not create %s\n,buf);
+ exit(0);
+ }
+ if (cache)
+#ifdef NATIVE_WIN32
+
snprintf(buf,BUFSIZ,%s\\%s_%s_%d.trace,cache,basefilename,dbname, queryid);
+#else
+
snprintf(buf,BUFSIZ,%s/%s_%s_%d.trace,cache,basefilename,dbname, queryid);
+#endif
+ else
+ snprintf(buf,BUFSIZ,%s_%s_%d.trace,basefilename,dbname,
queryid);
+ tachotrace= fopen(buf,w);
+ if( tachotrace == NULL){
+ fprintf(stderr,Could not create %s\n,buf);
+ exit(0);
+ }
+}
+
static void
progressBarInit(char *qry)
{
- char buf[BUFSIZ];
-
- snprintf(buf,BUFSIZ,%s_%s.json,basefilename,dbname);
- tachofd= fopen(buf,w);
- if( tachofd == NULL){
- fprintf(stderr,Could not create %s\n,buf);
- exit(0);
- }
- fprintf(tachofd,{ \tachograph\:0.1,\n);
- fprintf(tachofd,
\qid\:\%s\,\n,currentfunction?currentfunction:);
- fprintf(tachofd, \tag\:\%d\,\n,currenttag);
- fprintf(tachofd, \query\:\%s\,\n,qry);
- fprintf(tachofd, \started\: LLFMT,\n,starttime);
- fprintf(tachofd, \duration\:LLFMT,\n,duration);
- fprintf(tachofd, \instructions\:%d\n,malsize);
- fprintf(tachofd,},\n);
- fflush(tachofd);
+ fprintf(tachojson,{ \tachograph\:0.1,\n);
+ fprintf(tachojson,
\qid\:\%s\,\n,currentfunction?currentfunction:);
+ fprintf(tachojson, \tag\:\%d\,\n,currenttag);
+ fprintf(tachojson, \query\:\%s\,\n,qry);
+ fprintf(tachojson, \started\: LLFMT,\n,starttime);
+ fprintf(tachojson, \duration\:LLFMT,\n,duration);
+ fprintf(tachojson, \instructions\:%d\n,malsize);
+ fprintf(tachojson,},\n);
+ fflush(tachojson);
}
static void
@@ -516,6 +553,7 @@ update(EventRecord *ev)
}
if (debug)
fprintf(stderr, Enter function %s capture %d\n,
currentfunction, capturing);
+ initFiles();
return;
}
ev-clkticks -= starttime;
@@ -601,17 +639,17 @@ update(EventRecord *ev)
for( i=0; i malvartop;i++)
addSourcePair(nme,malvariables[i]);
}
- fprintf(tachofd,{\n);
-
MonetDB: default - use a 'subleftjoin' for foreignkey - join ind...
Changeset: 17fed0f3e977 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=17fed0f3e977
Modified Files:
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_prelude.c
monetdb5/optimizer/opt_prelude.h
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_support.h
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_gencode.c
sql/benchmarks/tpch/alter.sql
sql/benchmarks/tpch/load.sh
sql/include/sql_catalog.h
sql/include/sql_relation.h
sql/rel.txt
sql/server/rel_updates.c
Branch: default
Log Message:
use a 'subleftjoin' for foreignkey - join indices, solves reorder/hash problems
diffs (truncated from 352 to 300 lines):
diff --git a/monetdb5/optimizer/opt_mergetable.c
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -1491,6 +1491,17 @@ OPTmergetableImplementation(Client cntxt
actions++;
continue;
}
+ if (match 0 isMatLeftJoinOp(p) p-argc = 3 p-retc
== 2
+ match == 1 bats == 2) {
+ m = is_a_mat(getArg(p,p-retc), mat, mtop);
+ n = -1;
+
+ if (m = 0) {
+ mtop = mat_join2(mb, p, mat, mtop, m, n);
+ actions++;
+ continue;
+ }
+ }
/*
* Aggregate handling is a prime target for optimization.
* The simple cases are dealt with first.
diff --git a/monetdb5/optimizer/opt_prelude.c b/monetdb5/optimizer/opt_prelude.c
--- a/monetdb5/optimizer/opt_prelude.c
+++ b/monetdb5/optimizer/opt_prelude.c
@@ -107,6 +107,7 @@ str antijoinRef;
str bandjoinRef;
str thetajoinRef;
str subjoinRef;
+str subleftjoinRef;
str subantijoinRef;
str subbandjoinRef;
str subrangejoinRef;
@@ -358,6 +359,7 @@ void optimizerInit(void)
bandjoinRef = putName(bandjoin,8);
thetajoinRef = putName(thetajoin,9);
subjoinRef = putName(subjoin,7);
+ subleftjoinRef = putName(subleftjoin,11);
subantijoinRef = putName(subantijoin,11);
subbandjoinRef = putName(subbandjoin,11);
subrangejoinRef = putName(subrangejoin,12);
diff --git a/monetdb5/optimizer/opt_prelude.h b/monetdb5/optimizer/opt_prelude.h
--- a/monetdb5/optimizer/opt_prelude.h
+++ b/monetdb5/optimizer/opt_prelude.h
@@ -105,6 +105,7 @@ opt_export str antijoinRef;
opt_export str bandjoinRef;
opt_export str thetajoinRef;
opt_export str subjoinRef;
+opt_export str subleftjoinRef;
opt_export str subantijoinRef;
opt_export str subbandjoinRef;
opt_export str subrangejoinRef;
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -673,7 +673,9 @@ int isOrderby(InstrPtr p){
getFunctionId(p) == sortReverseRef);
}
-int isMatJoinOp(InstrPtr p){
+int
+isMatJoinOp(InstrPtr p)
+{
return (isSubJoin(p) || (getModuleId(p) == algebraRef
(getFunctionId(p) == crossRef ||
getFunctionId(p) == joinRef ||
@@ -685,6 +687,13 @@ int isMatJoinOp(InstrPtr p){
));
}
+int
+isMatLeftJoinOp(InstrPtr p)
+{
+ return (getModuleId(p) == algebraRef
+ getFunctionId(p) == subleftjoinRef);
+}
+
int isDelta(InstrPtr p){
return
(getModuleId(p)== sqlRef (
diff --git a/monetdb5/optimizer/opt_support.h b/monetdb5/optimizer/opt_support.h
--- a/monetdb5/optimizer/opt_support.h
+++ b/monetdb5/optimizer/opt_support.h
@@ -102,6 +102,7 @@ opt_export int isFragmentGroup(InstrPtr
opt_export int isFragmentGroup2(InstrPtr q);
opt_export int isDelta(InstrPtr q);
opt_export int isMatJoinOp(InstrPtr q);
+opt_export int isMatLeftJoinOp(InstrPtr q);
opt_export int isMapOp(InstrPtr q);
opt_export int isLikeOp(InstrPtr q);
opt_export int isTopn(InstrPtr q);
diff --git a/sql/backends/monet5/rel_bin.c b/sql/backends/monet5/rel_bin.c
--- a/sql/backends/monet5/rel_bin.c
+++ b/sql/backends/monet5/rel_bin.c
@@ -1424,7 +1424,7 @@ join_hash_key( mvc *sql, list *l )
}
static stmt *
-releqjoin( mvc *sql, list *l1, list *l2, int used_hash, comp_type cmp_op )
+releqjoin( mvc *sql, list *l1, list *l2, int used_hash, comp_type cmp_op, int
need_left )
{
node *n1 = l1-h, *n2 = l2-h;
stmt *l, *r, *res;
@@ -1432,7 +1432,10 @@ releqjoin( mvc *sql, list *l1, list *l2,
if (list_length(l1) = 1) {
l = l1-h-data;
r = l2-h-data;
- return stmt_join(sql-sa, l, r, cmp_op);
+ r = stmt_join(sql-sa, l, r, cmp_op);
+ if (need_left)
+ r-flag = cmp_left;
+ return r;
}
