Changeset: 45336a8d22d3 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=45336a8d22d3
Modified Files:
ChangeLog.Mar2018
clients/Tests/mclient-uri.SQL.bat
clients/Tests/mclient-uri.SQL.sh
clients/mapiclient/Tests/mclient--help.stable.err
clients/mapiclient/Tests/mclient--help.stable.err.Windows
clients/mapiclient/mclient.1
clients/mapiclient/mclient.c
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_logger.c
sql/test/BugDay_2005-10-06_2.9.3/Tests/alter_table_describe-mclient.SF-1146092.SQL.bat
sql/test/BugDay_2005-10-06_2.9.3/Tests/alter_table_describe-mclient.SF-1146092.SQL.sh
sql/test/BugTracker-2018/Tests/sqlitelogictest-group-by-having-in-count.Bug-6524.stable.err
sql/test/BugTracker-2018/Tests/sqlitelogictest-group-by-having-in-count.Bug-6524.stable.out
sql/test/pg_regress/Tests/float8.stable.err
testing/README
Branch: default
Log Message:
Merge with Mar2018 branch.
diffs (truncated from 811 to 300 lines):
diff --git a/ChangeLog.Mar2018 b/ChangeLog.Mar2018
--- a/ChangeLog.Mar2018
+++ b/ChangeLog.Mar2018
@@ -1,6 +1,32 @@
# ChangeLog file for devel
# This file is updated with Maddlog
+* Wed Feb 14 2018 Stefan Manegold <stefan.maneg...@cwi.nl>
+- mclient's executing time profiling options and output format have
+ been changed. Instead of implicitly via the "--interactive"/"-i"
+ option (or when using and interactive mclient console), execution time
+ profiling is now controlled via an explicit "--timer=timermode"/"-t
+ timermode" command-line option, or a "\t timermode" command in the
+ interactive mclient console. The default (also in the interactive
+ mclient console) is now timermode "none", i.e., no timing information
+ is given. Timermode "clock" activates client-side wall-clock timing
+ ("clk") in "human-friendly" format much like the interactive mode did
+ before. Timermode "performance" also provides detailed server-side
+ timings: "sql" is the time to parse the SQL query, optimize the
+ logical relational plan and create the initial physical (MAL) plan;
+ "opt" is the time to optimize the physical (MAL) plan; "run" is the
+ time to execute the physical (MAL) plan. With timermode "performance"
+ all server-side timings and the client-side wall-clock time are given
+ in milliseconds (ms). Note that the client-measured wall-clock time
+ "clk" is reported per query only when options "--interactive" or
+ "--echo" are used, because only then does mclient send individual
+ lines (statements) of the SQL script to the server. Otherwise, mclient
+ sends the SQL script in large(r) batch(es) to the server, and, thus,
+ only the total wall-clock time per batch is measured and reported. The
+ server-measured detailed performance timings "sql", "opt", "run" are
+ always measured and reported per query. Also, all timing information
+ is now given on a separate line and set to stderr rather than stdout.
+
* Fri Feb 9 2018 Sjoerd Mullender <sjo...@acm.org>
- Some types and constants were moved from configure (and hence
monetdb_config.h) to gdk.h. In particular, the types "lng" and
diff --git a/clients/Tests/mclient-uri.SQL.bat
b/clients/Tests/mclient-uri.SQL.bat
--- a/clients/Tests/mclient-uri.SQL.bat
+++ b/clients/Tests/mclient-uri.SQL.bat
@@ -3,4 +3,4 @@
@rem Windows doesn't do UNIX domain sockets, so only the one test here.
-mclient -d
"mapi:monetdb://%HOST%:%MAPIPORT%/%TSTDB%?language=sql&user=monetdb" -f test -E
utf-8 -s "select 1"
+mclient -d
"mapi:monetdb://%HOST%:%MAPIPORT%/%TSTDB%?language=sql&user=monetdb" -f test -t
none -E utf-8 -s "select 1"
diff --git a/clients/Tests/mclient-uri.SQL.sh b/clients/Tests/mclient-uri.SQL.sh
--- a/clients/Tests/mclient-uri.SQL.sh
+++ b/clients/Tests/mclient-uri.SQL.sh
@@ -3,7 +3,7 @@
# test the URI parsing capabilities of the MAPI library
Mlog "mclient -d
mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql&user=monetdb -f test -t none
-E utf-8 -s select 1"
-mclient -d "mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql&user=monetdb"
-f test -E utf-8 -s 'select 1'
+mclient -d "mapi:monetdb://$HOST:$MAPIPORT/$TSTDB?language=sql&user=monetdb"
-f test -t none -E utf-8 -s 'select 1'
Mlog "mclient -d
mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB&language=sql&user=monetdb
-f test -t none -E utf-8 -s select 1"
-mclient -d
"mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB&language=sql&user=monetdb"
-f test -E utf-8 -s 'select 1'
+mclient -d
"mapi:monetdb://$MAPIHOST/.s.monetdb.$MAPIPORT?database=$TSTDB&language=sql&user=monetdb"
-f test -t none -E utf-8 -s 'select 1'
diff --git a/clients/mapiclient/Tests/mclient--help.stable.err
b/clients/mapiclient/Tests/mclient--help.stable.err
--- a/clients/mapiclient/Tests/mclient--help.stable.err
+++ b/clients/mapiclient/Tests/mclient--help.stable.err
@@ -22,7 +22,7 @@ Options are:
-f kind | --format=kind specify output format
{csv,tab,raw,sql,xml,trash}
-H | --history load/save cmdline history (default off)
-i | --interactive interpret `\' commands on stdin
- -t | --timer=format use time formatting {clock,performance,none}
+ -t | --timer=format use time formatting {none,clock,performance}
(none is default)
-l language | --language=lang {sql,mal}
-L logfile | --log=logfile save client/server interaction
-s stmt | --statement=stmt run single statement
diff --git a/clients/mapiclient/Tests/mclient--help.stable.err.Windows
b/clients/mapiclient/Tests/mclient--help.stable.err.Windows
--- a/clients/mapiclient/Tests/mclient--help.stable.err.Windows
+++ b/clients/mapiclient/Tests/mclient--help.stable.err.Windows
@@ -22,7 +22,7 @@ Options are:
-f kind | --format=kind specify output format
{csv,tab,raw,sql,xml,trash}
-H | --history load/save cmdline history (default off)
-i | --interactive interpret `\' commands on stdin
- -t | --timer=format use time formatting {clock,performance,none}
+ -t | --timer=format use time formatting {none,clock,performance}
(none is default)
-l language | --language=lang {sql,mal}
-L logfile | --log=logfile save client/server interaction
-s stmt | --statement=stmt run single statement
diff --git a/clients/mapiclient/mclient.1 b/clients/mapiclient/mclient.1
--- a/clients/mapiclient/mclient.1
+++ b/clients/mapiclient/mclient.1
@@ -128,7 +128,7 @@ is assumed.
.TP
\fB\-\-database=\fP\fIdatabase\fP (\fB\-d\fP \fIdatabase\fP)
Specify the name or URI of the database to connect to.
-The \fB-d\fP can be omitted if an equally named file does not exist in
+The \fB\-d\fP can be omitted if an equally named file does not exist in
the current directory.
As such, the first non-option argument will be interpreted as database
to connect to if the argument does not exist as file.
@@ -156,15 +156,24 @@ See the section BACKSLASH COMMANDS below
.TP
\fB\-\-timer\fP\fB=\fP\fItimermode\fP (\fB\-t\fP \fItimermode\fP)
The \fItimer\fP command controls the format of the time reported for queries.
-The default mode is \fBclock\fP which reports on the wall-clock time in a
human friendly way.
-The timer mode \fBnone\fP turns off timing reporting.
-The timer mode \fBperformance\fP shows the timing components in millisecond
resolution.
+The default mode is \fBnone\fP which turns off timing reporting.
+The timer mode \fBclock\fP reports the client-side wall-clock time
+("\fBclk\fP") in a human-friendly format.
+The timer mode \fBperformance\fP reports client-side wall-clock time
+("\fBclk\fP") as well as detailed server-side timings, all in milliseconds
+(ms): the time to parse the SQL query, optimize the logical relational plan
+and create the initial physical (MAL) plan ("\fBsql\fP"); the time to
+optimize the physical (MAL) plan ("\fBopt\fP"); the time to execute the
+physical (MAL) plan ("\fBrun\fP").
+All timings are reported on stderr.
.br
-\fBNote\fP that Timermode \fBnone\fP (\fB--timer=none\fP) should be used with
formatting options
-.B csv,
-.B tab,
-.B xml,
-in order to not have the output(-format) compromized/invalidated by the extra
timing output.
+\fBNote\fP that the client-measured wall-clock time is reported per query
+\fBonly\fP when options \fB\-\-interactive\fP or \fB\-\-echo\fP are used,
+because only then does mclient send individual lines (statements) of the SQL
+script to the server. Otherwise, when mclient sends the SQL script in
+large(r) batch(es), only the total wall-clock time per batch is measured and
+reported. The server-measured detailed performance timings are always
+measured and reported per query.
.TP
\fB\-\-user\fP\fB=\fP\fIuser\fP (\fB\-u\fP \fIuser\fP)
Specify the user to connect as.
@@ -180,8 +189,9 @@ The possible values are
.BR csv ,
.BR tab ,
.BR raw ,
+.BR xml ,
and
-.BR xml .
+.BR trash .
.B csv
is comma-separated values,
.B tab
@@ -196,19 +206,17 @@ are clearly shown,
and
.B x
are synonyms and are another pretty format meant for human consumption
-where column values are printed in full and below each other, and
+where column values are printed in full and below each other,
.B xml
-is a valid (in the XML sense) document.
+is a valid (in the XML sense) document,
+and
+.B trash
+does not render any output, enabling performance measurements free of any
+output rendering/serialization costs.
In addition to plain \fBcsv\fP, two other forms are possible.
\fBcsv=\fP\fIc\fP uses \fIc\fP as column separator; \fBcsv+\fP\fIc\fP
uses \fIc\fP as column separator and produces a single header line in
addition to the data.
-.br
-\fBNote\fP that Timermode \fBnone\fP (\fB--timer=none\fP) should be used with
formatting options
-.B csv,
-.B tab,
-.B xml,
-in order to not have the output(-format) compromized/invalidated by the extra
timing output.
.TP
\fB\-\-echo\fP (\fB\-e\fP)
Echo the query.
@@ -432,7 +440,7 @@ is the name of the table,
.I $db
is the name of the database.
.PP
-mclient -d $db -s \(dqCOPY INTO $table FROM '$file' USING DELIMITERS
',','\e\en','\e\(dq'\(dq
+mclient \-d $db \-s \(dqCOPY INTO $table FROM '$file' USING DELIMITERS
',','\e\en','\e\(dq'\(dq
.PP
Efficiently import data from a CSV file into a table when the file is
to be read by mclient (e.g. the server has no access to the file).
@@ -443,7 +451,7 @@ is the name of the table,
.I $db
is the name of the database.
.PP
-mclient -d $db -s \(dqCOPY INTO $table FROM STDIN USING DELIMITERS
',','\e\en','\e\(dq'\(dq - < $file
+mclient \-d $db \-s \(dqCOPY INTO $table FROM STDIN USING DELIMITERS
',','\e\en','\e\(dq'\(dq \- < $file
.PP
Note that in this latter case, if a count of records is supplied, it
should be at least as large as the number of records actually present
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -72,7 +72,6 @@ static char *language = NULL;
static char *logfile = NULL;
static char promptbuf[16];
static int echoquery = 0;
-static int showtiming = 0;
#ifdef HAVE_ICONV
static char *encoding;
#endif
@@ -101,7 +100,6 @@ int csvheader = 0; /* include header li
/* use a 64 bit integer for the timer */
typedef int64_t timertype;
-#define TTFMT "%" PRId64
static timertype t0, t1; /* used for timing */
@@ -256,42 +254,73 @@ timerHumanStop(void)
}
static enum itimers {
- T_CLOCK = 0, // render wallclock time in human readable format
- T_PERF, // return detailed performance
- T_NONE // don't render the timing information
-} timermode = T_CLOCK;
+ T_NONE = 0, // don't render the timing information
+ T_CLOCK, // render wallclock time in human readable format
+ T_PERF // return detailed performance
+} timermode = T_NONE;
-static char htimbuf[128];
-static char *
-timerHuman(int64_t sqloptimizer, int64_t maloptimizer, int64_t querytime)
+static int timerHumanCalled = 0;
+static void
+timerHuman(int64_t sqloptimizer, int64_t maloptimizer, int64_t querytime, int
singleinstr, int total)
{
timertype t = th - t0;
- if (timermode == T_CLOCK) {
- if (t / 1000 < 950) {
- snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%03d
ms" , t / 1000, (int) (t % 1000));
- return htimbuf;
+ timerHumanCalled = 1;
+
+ /*
+ * report only the times we do actually measure:
+ * - client-measured wall-clock time per query only when executing
indivual queries,
+ * otherwise only the total wall-clock time at the end of a batch;
+ * - server-measured detailed performance measures only per query.
+ */
+
+ /* (!singleinstr != !total) is C for ((singleinstr != 0) XOR (total !=
0)) */
+ if (timermode == T_CLOCK && (!singleinstr != !total)) {
+ /* print wall-clock in "human-friendly" format */
+ fflush(stderr);
+ mnstr_flush(toConsole);
+ if (t / 1000 < 1000) {
+ fprintf(stderr, "clk: %" PRId64 ".%03d ms\n", t / 1000,
(int) (t % 1000));
+ fflush(stderr);
+ return;
}
t /= 1000;
if (t / 1000 < 60) {
- snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ".%02d
sec", t / 1000, (int) ((t % 1000) / 100));
- return htimbuf;
+ fprintf(stderr, "clk: %" PRId64 ".%03d sec\n", t /
1000, (int) (t % 1000));
+ fflush(stderr);
+ return;
}
t /= 1000;
- snprintf(htimbuf, sizeof(htimbuf), "clk: " TTFMT ":%02d min", t
/ 60, (int) (t % 60));
- return htimbuf;
+ if (t / 60 < 60) {
+ fprintf(stderr, "clk: %" PRId64 ":%02d min\n", t / 60,
(int) (t % 60));
+ fflush(stderr);
+ return;
+ }
+ t /= 60;
+ fprintf(stderr, "clk: %" PRId64 ":%02d h\n", t / 60, (int) (t %
60));
+ fflush(stderr);
+ return;
}
- /* for performance measures we use milliseconds as the base */
if (timermode == T_PERF) {
- snprintf(htimbuf, sizeof(htimbuf), "clk:%" PRId64 ".%03d sql:%"
PRId64 ".%03d opt:%" PRId64 ".%03d run:%" PRId64 ".%03d ms",
- t / 1000, (int) (t % 1000),
- sqloptimizer / 1000, (int) (sqloptimizer % 1000),
- maloptimizer / 1000, (int) (maloptimizer % 1000),
- querytime / 1000, (int) (querytime % 1000));
- return htimbuf;
+ /* for performance measures we use milliseconds as the base */
+ if ((!singleinstr != !total) || !total) {
+ fflush(stderr);
+ mnstr_flush(toConsole);
+ }
+ if (!total)
+ fprintf(stderr, "sql:%" PRId64 ".%03d opt:%" PRId64
".%03d run:%" PRId64 ".%03d ",
+ sqloptimizer / 1000, (int) (sqloptimizer %
1000),
+ maloptimizer / 1000, (int) (maloptimizer %
1000),
+ querytime / 1000, (int) (querytime % 1000));
+ if (!singleinstr != !total)
+ fprintf(stderr, "clk:%" PRId64 ".%03d ", t / 1000,
(int) (t % 1000));
+ if ((!singleinstr != !total) || !total) {
+ fprintf(stderr, "ms\n");
+ fflush(stderr);
+ }
+ return;
}
- htimbuf[0] = 0;
- return htimbuf;
+ return;
}
/* The Mapi library eats away the comment lines, which we need to
@@ -1116,7 +1145,6 @@ TESTrenderer(MapiHdl hdl)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
