MonetDB: default - r tests using mfilter variable output now
Changeset: 5d432332635f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d432332635f Modified Files: clients/R/Tests/dbi.stable.err clients/R/Tests/deps-install.R clients/R/Tests/deps-test.R clients/R/Tests/install.R Branch: default Log Message: r tests using mfilter variable output now diffs (85 lines): diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err --- a/clients/R/Tests/dbi.stable.err +++ b/clients/R/Tests/dbi.stable.err @@ -37,6 +37,7 @@ Use 'dbIsValid' instead. See help("Deprecated") Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. +Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Treating character vector parameter as file name(s) for monetdb.read.csv() Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. diff --git a/clients/R/Tests/deps-install.R b/clients/R/Tests/deps-install.R --- a/clients/R/Tests/deps-install.R +++ b/clients/R/Tests/deps-install.R @@ -1,8 +1,15 @@ # autoinstall DBI and digest, we need those to install MonetDB.R -dd <- capture.output(suppressMessages(suppressWarnings({ - repos <- "http://cran.rstudio.com/"; - lp <- c("devtools", "digest") - install.packages(lp, repos=repos, quiet=T) - update.packages(repos=repos, ask=F, oldPkgs=lp) - devtools::install_github("rstats-db/DBI", quiet=T) -}))) + +cat("#~BeginProfilingOutput~#\n", file=stderr()) +cat("#~BeginProfilingOutput~#\n", file=stdout()) + +repos <- "http://cran.rstudio.com/"; +packages_required <- c("devtools", "digest") +install.packages(packages_required, repos=repos, quiet=T) +update.packages(repos=repos, ask=F, oldPkgs=packages_required) +devtools::install_github("rstats-db/DBI", quiet=T) + +cat("#~EndProfilingOutput~#\n", file=stderr()) +cat("#~EndProfilingOutput~#\n\n", file=stdout()) + +stopifnot(all(c(packages_required, "DBI") %in% installed.packages()[,"Package"])) diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R --- a/clients/R/Tests/deps-test.R +++ b/clients/R/Tests/deps-test.R @@ -1,8 +1,21 @@ -dd <- capture.output(suppressMessages(suppressWarnings({ - (function(lp) { +packages_required <- c('Rcpp', 'dplyr', 'Lahman', 'nycflights13') + +install_or_upgrade_packages <- function(lp) { np <- lp[!(lp %in% installed.packages()[,"Package"])] repos <- 'http://cran.rstudio.com/' if(length(np)) install.packages(np, repos=repos, quiet=T) update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T) - })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata')) -}))) +} + + +cat("#~BeginProfilingOutput~#\n", file=stderr()) +cat("#~BeginProfilingOutput~#\n", file=stdout()) + +devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T) + +install_or_upgrade_packages(packages_required) + +cat("#~EndProfilingOutput~#\n", file=stderr()) +cat("#~EndProfilingOutput~#\n", file=stdout()) + +stopifnot(all(packages_required %in% installed.packages()[,"Package"])) diff --git a/clients/R/Tests/install.R b/clients/R/Tests/install.R --- a/clients/R/Tests/install.R +++ b/clients/R/Tests/install.R @@ -1,4 +1,11 @@ -dd <- capture.output(suppressMessages( { - devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T) -})) -library(MonetDBLite,quietly=T) +cat("#~BeginProfilingOutput~#\n", file=stderr()) +cat("#~BeginProfilingOutput~#\n", file=stdout()) + +devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T) + +cat("#~EndProfilingOutput~#\n", file=stderr()) +cat("#~EndProfilingOutput~#\n", file=stdout()) + +stopifnot("MonetDBLite" %in% installed.packages()[,"Package"]) + +library(MonetDBLite, quietly=T) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - its been several years since clang liked the ...
Changeset: d432f9c891fb for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d432f9c891fb Modified Files: configure.ag Branch: default Log Message: its been several years since clang liked the -pthread flag diffs (12 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -1450,7 +1450,7 @@ AS_VAR_IF([have_pthread], [no], [], [ save_LIBS="$LIBS" save_CPPFLAGS="$CPPFLAGS" AS_CASE([$GCC-$have_pthread-$CC_ver], - [yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*|yes-auto-clang-8.*|yes-yes-clang-8.*], [ + [yes-auto-clang-*|yes-yes-clang-*], [ # clang 5.*/6.*/7.* (Xcode 6.0) does not # seem to have / require -pthread as compiler # option; on Mac OS X Yosamite, "Apple LLVM ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: queryid - added query id export to monet
Changeset: 0c4024427f78 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c4024427f78 Modified Files: sql/backends/monet5/mal_backend.h sql/backends/monet5/sql.c sql/backends/monet5/sql_result.c Branch: queryid Log Message: added query id export to monet diffs (75 lines): diff --git a/sql/backends/monet5/mal_backend.h b/sql/backends/monet5/mal_backend.h --- a/sql/backends/monet5/mal_backend.h +++ b/sql/backends/monet5/mal_backend.h @@ -38,7 +38,7 @@ typedef struct backend { stream *out; ofmtoutput_format; /* csv, json */ Client client; - MalBlkPtr mb; /* needed during mal generation */ + MalBlkPtr mb; /* needed during mal generation and result set export */ int mvc_var; int vtop; /* top of the variable stack before the current function */ cq *q; /* pointer to the cached query */ diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -2825,6 +2825,9 @@ mvc_result_set_wrap( Client cntxt, MalBl BBPunfix(bid); } /* now sent it to the channel cntxt->fdout */ + + ((backend*)(cntxt->sqlcontext))->mb = mb; + if (mvc_export_result(cntxt->sqlcontext, cntxt->fdout, res)) msg = createException(SQL, "sql.resultset", "failed"); wrapup_result_set: @@ -3290,6 +3293,9 @@ mvc_affected_rows_wrap(Client cntxt, Mal assert(mtype == TYPE_lng); nr = *getArgReference_lng(stk, pci, 2); b = cntxt->sqlcontext; + + b->mb = mb; // to allow query id export in result set + error = mvc_export_affrows(b, b->out, nr, ""); if (error) throw(SQL, "sql.affectedRows", "failed"); @@ -3401,6 +3407,8 @@ mvc_scalar_value_wrap(Client cntxt, MalB if (ATOMextern(mtype)) p = *(ptr *) p; + b->mb = mb; // for query id export + // scalar values are single-column result sets mvc_result_table(b->mvc, 1, 1, NULL); mvc_result_value(b->mvc, *tn, *cn, *type, *digits, *scale, p, mtype); diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c --- a/sql/backends/monet5/sql_result.c +++ b/sql/backends/monet5/sql_result.c @@ -1610,7 +1610,7 @@ mvc_export_value(backend *b, stream *s, assert(qtype == Q_TABLE); if (csv && - (mnstr_write(s, "&1 0 1 1 1\n", 11, 1) != 1 || + (mnstr_printf(s, "&1 0 1 1 1 "OIDFMT"\n", b->mb->tag) != 1 || /* fallback to default tuplecount (1) and id (0) */ /* TODO first header name then values */ mnstr_write(s, "% ", 2, 1) != 1 || @@ -1681,7 +1681,7 @@ mvc_export_affrows(backend *b, stream *s m->rowcnt = val; stack_set_number(m, "rowcnt", m->rowcnt); - if (mnstr_write(s, "&2 ", 3, 1) != 1 || !mvc_send_lng(s, val) || mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, m->last_id) || mnstr_write(s, "\n", 1, 1) != 1) + if (mnstr_write(s, "&2 ", 3, 1) != 1 || !mvc_send_lng(s, val) || mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, m->last_id) || mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, (lng) b->mb->tag) || mnstr_write(s, "\n", 1, 1) != 1) return -1; if (mvc_export_warning(s, w) != 1) return -1; @@ -1741,6 +1741,9 @@ mvc_export_head(backend *b, stream *s, i if (!mvc_send_int(s, (m->reply_size >= 0 && (BUN) m->reply_size < count) ? m->reply_size : (int) count)) return -1; + if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, (lng) b->mb->tag)) + return -1; + if (mnstr_write(s, "\n% ", 3, 1) != 1) return -1; for (i = 0; i < t->nr_cols; i++) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Smaller test for Bug 6178
Changeset: 41e0ace65aed for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=41e0ace65aed Added Files: sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out Modified Files: sql/test/BugTracker-2017/Tests/All Branch: Dec2016 Log Message: Smaller test for Bug 6178 diffs (171 lines): diff --git a/sql/test/BugTracker-2017/Tests/All b/sql/test/BugTracker-2017/Tests/All --- a/sql/test/BugTracker-2017/Tests/All +++ b/sql/test/BugTracker-2017/Tests/All @@ -1,1 +1,2 @@ shutdown.Bug-6182 +avggroupbysq.Bug-6178 diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql @@ -0,0 +1,19 @@ +START TRANSACTION; + +CREATE TABLE x (tailnum STRING, arr_delay INTEGER); + +COPY 2 RECORDS INTO x FROM STDIN USING DELIMITERS ',','\n','"' NULL as ''; +"N907MQ", +"N907MQ",191 + +SELECT * FROM x; +SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum; + +-- these two should be the same?! +-- N907MQ|NULL is in the data, hence N907MQ|NULL should also be the result + +SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY tailnum; +SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) AS xxx WHERE tailnum = 'N907MQ'; + + + diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err @@ -0,0 +1,35 @@ +stderr of test 'avggroupbysq.Bug-6178` in directory 'sql/test/BugTracker-2017` itself: + + +# 18:20:38 > +# 18:20:38 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36572" "--set" "mapi_usock=/var/tmp/mtest-8698/.s.monetdb.36572" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017" "--set" "embedded_py=true" +# 18:20:38 > + +# builtin opt gdk_dbpath = /Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 36572 +# cmdline opt mapi_usock = /var/tmp/mtest-8698/.s.monetdb.36572 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017 +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 18:20:38 > +# 18:20:38 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-8698" "--port=36572" +# 18:20:38 > + + +# 18:20:39 > +# 18:20:39 > "Done." +# 18:20:39 > + diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out @@ -0,0 +1,96 @@ +stdout of test 'avggroupbysq.Bug-6178` in directory 'sql/test/BugTracker-2017` itself: + + +# 18:20:38 > +# 18:20:38 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36572" "--set" "mapi_usock=/var/tmp/mtest-8698/.s.monetdb.36572" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017" "--set" "embedded_py=true" +# 18:20:38 > + +# MonetDB 5 server v11.25.4 +# This is an unreleased version +# Serving database 'mTests_sql_test_BugTracker-2017', using 4 threads +# Compiled for x86_64-apple-darwin16.3.0/64bit with 128bit integers +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36572/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-8698/.s.monetdb.36572 +# MonetDB/SQL module loaded + + + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql scrip
MonetDB: Dec2016 - Update for Bug 6178, issue only appears when ...
Changeset: a9ebbd5b2e53 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a9ebbd5b2e53 Modified Files: sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out Branch: Dec2016 Log Message: Update for Bug 6178, issue only appears when using AVG diffs (92 lines): diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql --- a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql +++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql @@ -6,14 +6,22 @@ COPY 2 RECORDS INTO x FROM STDIN USING D "N907MQ", "N907MQ",191 -SELECT * FROM x; +-- correct result, 191 +SELECT AVG( arr_delay ) FROM x; + +-- wrong result, NULL SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum; --- these two should be the same?! --- N907MQ|NULL is in the data, hence N907MQ|NULL should also be the result +-- works fine with MIN +SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum; +-- correct again SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY tailnum; + +-- wrong again SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) AS xxx WHERE tailnum = 'N907MQ'; +-- both work fine with MIN +SELECT tailnum , MIN( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY tailnum; +SELECT * FROM ( SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum ) AS xxx WHERE tailnum = 'N907MQ'; - diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out --- a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out +++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out @@ -64,31 +64,48 @@ Ready. #"N907MQ", #"N907MQ",191 [ 2] -#SELECT * FROM x; -% sys.x, sys.x # table_name -% tailnum, arr_delay # name -% clob,int # type -% 6, 3 # length -[ "N907MQ",NULL] -[ "N907MQ",191 ] +#SELECT AVG( arr_delay ) FROM x; +% sys.L4 # table_name +% L3 # name +% double # type +% 24 # length +[ 191 ] #SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum; % sys.x, sys.L4 # table_name % tailnum, L3 # name % clob,double # type % 6, 24 # length -[ "N907MQ",NULL] +[ "N907MQ",191 ] +#SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum; +% sys.x, sys.L4 # table_name +% tailnum, L3 # name +% clob,int # type +% 6, 3 # length +[ "N907MQ",191 ] #SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY tailnum; % sys.x, sys.L4 # table_name % tailnum, L3 # name % clob,double # type % 6, 24 # length -[ "N907MQ",NULL] +[ "N907MQ",191 ] #SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) AS xxx WHERE tailnum = 'N907MQ'; % sys.xxx, sys.xxx # table_name % tailnum, L3 # name % clob,double # type % 6, 24 # length -[ "N907MQ",NULL] +[ "N907MQ",191 ] +#SELECT tailnum , MIN( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY tailnum; +% sys.x, sys.L4 # table_name +% tailnum, L3 # name +% clob,int # type +% 6, 3 # length +[ "N907MQ",191 ] +#SELECT * FROM ( SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum ) AS xxx WHERE tailnum = 'N907MQ'; +% sys.xxx, sys.xxx # table_name +% tailnum, L3 # name +% clob,int # type +% 6, 3 # length +[ "N907MQ",191 ] # 18:20:39 > # 18:20:39 > "Done." ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Fix for Bug 6178
Changeset: c10ab2216eec for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c10ab2216eec Modified Files: gdk/gdk_aggr.c monetdb5/optimizer/opt_mergetable.c Branch: Dec2016 Log Message: Fix for Bug 6178 diffs (27 lines): diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -1386,7 +1386,7 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT return GDK_SUCCEED; } - if ((e == NULL || + if (!skip_nils && (e == NULL || (BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) && (BATtdense(g) || (g->tkey && g->tnonil))) { /* trivial: singleton groups, so all results are equal diff --git a/monetdb5/optimizer/opt_mergetable.c b/monetdb5/optimizer/opt_mergetable.c --- a/monetdb5/optimizer/opt_mergetable.c +++ b/monetdb5/optimizer/opt_mergetable.c @@ -1107,10 +1107,7 @@ mat_group_aggr(MalBlkPtr mb, InstrPtr p, ai2 = pushArgument(mb, ai2, getArg(ai1, 0)); ai2 = pushArgument(mb, ai2, mat[g].mv); ai2 = pushArgument(mb, ai2, mat[e].mv); - if (isAvg) - ai2 = pushBit(mb, ai2, 0); /* do not skip nils */ - else - ai2 = pushBit(mb, ai2, 1); /* skip nils */ + ai2 = pushBit(mb, ai2, 1); /* skip nils */ if (getFunctionId(p) != subminRef && getFunctionId(p) != submaxRef) ai2 = pushBit(mb, ai2, 1); pushInstruction(mb, ai2); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Allow aggregation count optimization in more ...
Changeset: bf160289ab50 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bf160289ab50 Modified Files: gdk/gdk_aggr.c Branch: Dec2016 Log Message: Allow aggregation count optimization in more cases on Sjoerd's request diffs (12 lines): diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c --- a/gdk/gdk_aggr.c +++ b/gdk/gdk_aggr.c @@ -1386,7 +1386,7 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT return GDK_SUCCEED; } - if (!skip_nils && (e == NULL || + if ((!skip_nils || cntsp == NULL || b->tnonil) && (e == NULL || (BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) && (BATtdense(g) || (g->tkey && g->tnonil))) { /* trivial: singleton groups, so all results are equal ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - Check for malloc failures in several places
Changeset: 039e1a5a4266 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=039e1a5a4266 Modified Files: monetdb5/mal/mal_builder.c monetdb5/mal/mal_instruction.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_stack.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_reorder.c sql/backends/monet5/sql_optimizer.c Branch: Dec2016 Log Message: Check for malloc failures in several places diffs (141 lines): diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c --- a/monetdb5/mal/mal_builder.c +++ b/monetdb5/mal/mal_builder.c @@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val } cst.len= (int) strlen(cst.val.sval); getArg(q,0) = defConstant(mb,TYPE_str,&cst); + if (getArg(q,0) < 0) { + freeInstruction(q); + return NULL; + } clrVarConstant(mb,getArg(q,0)); setVarDisabled(mb,getArg(q,0)); if (mb->errors) { diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -730,7 +730,6 @@ makeVarSpace(MalBlkPtr mb) if (new == NULL) { mb->errors++; showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no storage left\n"); - assert(0); return -1; } memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr)); @@ -1258,6 +1257,9 @@ defConstant(MalBlkPtr mb, int type, ValP return k; } k = newTmpVariable(mb, type); + if (k == -1) { + return k; + } setVarConstant(mb, k); setVarFixed(mb, k); if (type >= 0 && type < GDKatomcnt && ATOMextern(type)) diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c --- a/monetdb5/mal/mal_interpreter.c +++ b/monetdb5/mal/mal_interpreter.c @@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size) assert(size >= mb->vsize); stk = newGlobalStack(size); + if (!stk) { + return NULL; + } //memset((char *)stk, 0, stackSize(size)); already set //stk->stksize = size; stk->stktop = mb->vtop; diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c --- a/monetdb5/mal/mal_stack.c +++ b/monetdb5/mal/mal_stack.c @@ -59,8 +59,9 @@ newGlobalStack(int size) MalStkPtr s; s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk)); - if (s == NULL) - GDKfatal("newGlobalStack:can not obtain memory\n"); + if (!s) { + return NULL; + } s->stksize = size; return s; } @@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn return old; k = ((cnt / STACKINCR) + 1) * STACKINCR; s = newGlobalStack(k); + if (!s) { + return NULL; + } memcpy(s, old, stackSize(old->stksize)); s->stksize = k; GDKfree(old); diff --git a/monetdb5/optimizer/opt_mergetable.c b/monetdb5/optimizer/opt_mergetable.c --- a/monetdb5/optimizer/opt_mergetable.c +++ b/monetdb5/optimizer/opt_mergetable.c @@ -1647,8 +1647,9 @@ OPTmergetableImplementation(Client cntxt ml.vsize = mb->vsize; ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); - if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) + if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) { goto cleanup; + } for (i=0; i 0){ chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE); diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c --- a/monetdb5/optimizer/opt_reorder.c +++ b/monetdb5/optimizer/opt_reorder.c @@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr sz += list[i]->used; } uselist = GDKzalloc(sizeof(int)*sz); + if (!uselist) { + GDKfree(list); + GDKfree(var); + return NULL; + } for(i=0;istop; i++) { if (list[i]->cnt) { diff --git a/sql/backends/monet5/sql_optimizer.c b/sql/backends/monet5/sql_optimizer.c --- a/sql/backends/monet5/sql_optimizer.c +++ b/sql/backends/monet5/sql_optimizer.c @@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu return size; } -static lng +static lng SQLgetSpace(mvc *m, MalBlkPtr mb, int prepare) { sql_trans *tr = m->session->tr; - lng size,space = 0, i; + lng size,space = 0, i; for (i = 0; i < mb->stop; i++) { InstrPtr p = mb->stmt[i]; @@ -80,7 +80,7 @@ SQLgetSpace(mvc *m, MalBlkPtr mb, int pr sql_table *t = 0; sql_column *c = 0; - if (!s || strcmp(s->base.
MonetDB: default - new test case for malloc failures
Changeset: 82dad11b6fd3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=82dad11b6fd3 Added Files: sql/test/malloc_fail/Tests/mallocs.SQL.py Removed Files: sql/test/malloc_fail/Tests/initialize.sql sql/test/malloc_fail/Tests/initialize.stable.err sql/test/malloc_fail/Tests/initialize.stable.out sql/test/malloc_fail/Tests/setmemorylimit-fail.reqtests sql/test/malloc_fail/Tests/setmemorylimit-fail.sql sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out sql/test/malloc_fail/Tests/setmemorylimit-fail2.reqtests sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql sql/test/malloc_fail/Tests/setmemorylimit.reqtests sql/test/malloc_fail/Tests/setmemorylimit.sql sql/test/malloc_fail/Tests/setmemorylimit.stable.err sql/test/malloc_fail/Tests/setmemorylimit.stable.out Modified Files: gdk/gdk_utils.c gdk/gdk_utils.h monetdb5/mal/mal_exception.c monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/mal_io.h monetdb5/modules/mal/mal_io.mal sql/test/malloc_fail/Tests/All Branch: default Log Message: new test case for malloc failures diffs (truncated from 527 to 300 lines): diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c --- a/gdk/gdk_utils.c +++ b/gdk/gdk_utils.c @@ -319,7 +319,7 @@ int GDK_vm_trim = 1; #include "gdk_atomic.h" static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0; #ifndef NDEBUG -static volatile lng GDK_mallocedbytes_limit = -1; +static volatile lng GDK_malloc_success_count = -1; #endif static volatile ATOMIC_TYPE GDK_vm_cursize = 0; #ifdef GDK_VM_KEEPHISTO @@ -1639,12 +1639,17 @@ GDKmalloc_prefixsize(size_t size) return s; } +#ifndef NDEBUG +static MT_Lock mallocsuccesslock; +#endif + void -GDKsetmemorylimit(lng nbytes) +GDKsetmallocsuccesscount(lng count) { - (void) nbytes; + (void) count; #ifndef NDEBUG - GDK_mallocedbytes_limit = nbytes; + MT_lock_init(&mallocsuccesslock, "mallocsuccesslock"); + GDK_malloc_success_count = count; #endif } @@ -1668,10 +1673,15 @@ GDKmallocmax(size_t size, size_t *maxsiz } #ifndef NDEBUG /* fail malloc for testing purposes depending on set limit */ - if (GDK_mallocedbytes_limit >= 0 && - size > (size_t) GDK_mallocedbytes_limit) { + if (GDK_malloc_success_count > 0) { + MT_lock_set(&mallocsuccesslock); + if (GDK_malloc_success_count > 0) GDK_malloc_success_count--; + MT_lock_unset(&mallocsuccesslock); + } + if (GDK_malloc_success_count == 0) { return NULL; } + #endif size = (size + 7) & ~7; /* round up to a multiple of eight */ s = GDKmalloc_prefixsize(size); diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h --- a/gdk/gdk_utils.h +++ b/gdk/gdk_utils.h @@ -77,7 +77,7 @@ gdk_export void MT_init(void);/* init gdk_export int GDKinit(opt *set, int setlen); /* used for testing only */ -gdk_export void GDKsetmemorylimit(lng nbytes); +gdk_export void GDKsetmallocsuccesscount(lng count); /* * Upon closing the session, all persistent BATs should be saved and diff --git a/monetdb5/mal/mal_exception.c b/monetdb5/mal/mal_exception.c --- a/monetdb5/mal/mal_exception.c +++ b/monetdb5/mal/mal_exception.c @@ -57,7 +57,10 @@ createExceptionInternal(enum malexceptio { char *message; int len; - + // if there is an error we allow memory allocation once again +#ifndef NDEBUG + GDKsetmallocsuccesscount(-1); +#endif message = GDKmalloc(GDKMAXERRLEN); if (message == NULL) return M5OutOfMemory; /* last resort */ diff --git a/monetdb5/modules/mal/mal_io.c b/monetdb5/modules/mal/mal_io.c --- a/monetdb5/modules/mal/mal_io.c +++ b/monetdb5/modules/mal/mal_io.c @@ -779,8 +779,8 @@ IOimport(void *ret, bat *bid, str *fnme) str -IOsetmemorylimit(void *res, lng *nbytes) { +IOsetmallocsuccesscount(void *res, lng *count) { (void) res; - GDKsetmemorylimit(*nbytes); + GDKsetmallocsuccesscount(*count); return MAL_SUCCEED; } diff --git a/monetdb5/modules/mal/mal_io.h b/monetdb5/modules/mal/mal_io.h --- a/monetdb5/modules/mal/mal_io.h +++ b/monetdb5/modules/mal/mal_io.h @@ -29,6 +29,6 @@ mal_export str io_stdin(Client cntxt, Ma mal_export str io_stdout(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); mal_export str io_stderr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); -mal_export str IOsetmemorylimit(void *res, lng *nbytes); +mal_export str IOsetmallocsuccesscount(void *res, lng *nbytes); #endif /* _PRINT_H_ */ diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal --- a/monetdb5/modules/mal/mal_io.mal +++ b/monetdb5/modules/mal/mal_io.mal @@ -62,6 +62,6 @@ comment "Import a BAT from an ASCII dump else parsi
MonetDB: default - Merge with Dec2016
Changeset: 7aee4214df1a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7aee4214df1a Modified Files: monetdb5/mal/mal_builder.c monetdb5/mal/mal_instruction.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_stack.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_reorder.c sql/backends/monet5/sql_optimizer.c Branch: default Log Message: Merge with Dec2016 diffs (141 lines): diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c --- a/monetdb5/mal/mal_builder.c +++ b/monetdb5/mal/mal_builder.c @@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val } cst.len= (int) strlen(cst.val.sval); getArg(q,0) = defConstant(mb,TYPE_str,&cst); + if (getArg(q,0) < 0) { + freeInstruction(q); + return NULL; + } clrVarConstant(mb,getArg(q,0)); setVarDisabled(mb,getArg(q,0)); if (mb->errors) { diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -717,7 +717,6 @@ makeVarSpace(MalBlkPtr mb) if (new == NULL) { mb->errors++; showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no storage left\n"); - assert(0); return -1; } memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr)); @@ -1245,6 +1244,9 @@ defConstant(MalBlkPtr mb, int type, ValP return k; } k = newTmpVariable(mb, type); + if (k == -1) { + return k; + } setVarConstant(mb, k); setVarFixed(mb, k); if (type >= 0 && type < GDKatomcnt && ATOMextern(type)) diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c --- a/monetdb5/mal/mal_interpreter.c +++ b/monetdb5/mal/mal_interpreter.c @@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size) assert(size >= mb->vsize); stk = newGlobalStack(size); + if (!stk) { + return NULL; + } //memset((char *)stk, 0, stackSize(size)); already set //stk->stksize = size; stk->stktop = mb->vtop; diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c --- a/monetdb5/mal/mal_stack.c +++ b/monetdb5/mal/mal_stack.c @@ -59,8 +59,9 @@ newGlobalStack(int size) MalStkPtr s; s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk)); - if (s == NULL) - GDKfatal("newGlobalStack:can not obtain memory\n"); + if (!s) { + return NULL; + } s->stksize = size; return s; } @@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn return old; k = ((cnt / STACKINCR) + 1) * STACKINCR; s = newGlobalStack(k); + if (!s) { + return NULL; + } memcpy(s, old, stackSize(old->stksize)); s->stksize = k; GDKfree(old); diff --git a/monetdb5/optimizer/opt_mergetable.c b/monetdb5/optimizer/opt_mergetable.c --- a/monetdb5/optimizer/opt_mergetable.c +++ b/monetdb5/optimizer/opt_mergetable.c @@ -1544,8 +1544,9 @@ OPTmergetableImplementation(Client cntxt ml.vsize = mb->vsize; ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); - if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) + if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) { goto cleanup; + } for (i=0; i 0){ chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE); diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c --- a/monetdb5/optimizer/opt_reorder.c +++ b/monetdb5/optimizer/opt_reorder.c @@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr sz += list[i]->used; } uselist = GDKzalloc(sizeof(int)*sz); + if (!uselist) { + GDKfree(list); + GDKfree(var); + return NULL; + } for(i=0;istop; i++) { if (list[i]->cnt) { diff --git a/sql/backends/monet5/sql_optimizer.c b/sql/backends/monet5/sql_optimizer.c --- a/sql/backends/monet5/sql_optimizer.c +++ b/sql/backends/monet5/sql_optimizer.c @@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu return size; } -static lng +static lng SQLgetSpace(mvc *m, MalBlkPtr mb, int prepare) { sql_trans *tr = m->session->tr; - lng size,space = 0, i; + lng size,space = 0, i; for (i = 0; i < mb->stop; i++) { InstrPtr p = mb->stmt[i]; @@ -80,7 +80,7 @@ SQLgetSpace(mvc *m, MalBlkPtr mb, int pr sql_table *t = 0; sql_column *c = 0; - if (!s || strcmp(s->base.name, dt_schema) == 0) +
MonetDB: default - moar spaces
Changeset: c37ce964753c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c37ce964753c Modified Files: sql/test/malloc_fail/Tests/mallocs.SQL.py Branch: default Log Message: moar spaces diffs (33 lines): diff --git a/sql/test/malloc_fail/Tests/mallocs.SQL.py b/sql/test/malloc_fail/Tests/mallocs.SQL.py --- a/sql/test/malloc_fail/Tests/mallocs.SQL.py +++ b/sql/test/malloc_fail/Tests/mallocs.SQL.py @@ -10,19 +10,19 @@ cursor = dbh.cursor() q = """ create procedure setmallocsuccesscount(count BIGINT) - external name "io"."setmallocsuccesscount"; +external name "io"."setmallocsuccesscount"; call setmallocsuccesscount(%d); SELECT * FROM tables; """ i = 3300 while i > 1000: - #print(i) - i-=1 - try: - cursor.execute(q % (i)) - except Exception as e: - #print(e) - pass - finally: - dbh.rollback() +#print(i) +i-=1 +try: +cursor.execute(q % (i)) +except Exception as e: +#print(e) +pass +finally: +dbh.rollback() ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - merge
Changeset: d314807446e7 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d314807446e7 Modified Files: monetdb5/mal/mal_builder.c monetdb5/mal/mal_instruction.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_stack.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_reorder.c sql/backends/monet5/sql_optimizer.c Branch: Dec2016 Log Message: merge diffs (141 lines): diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c --- a/monetdb5/mal/mal_builder.c +++ b/monetdb5/mal/mal_builder.c @@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val } cst.len= (int) strlen(cst.val.sval); getArg(q,0) = defConstant(mb,TYPE_str,&cst); + if (getArg(q,0) < 0) { + freeInstruction(q); + return NULL; + } clrVarConstant(mb,getArg(q,0)); setVarDisabled(mb,getArg(q,0)); if (mb->errors) { diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -730,7 +730,6 @@ makeVarSpace(MalBlkPtr mb) if (new == NULL) { mb->errors++; showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no storage left\n"); - assert(0); return -1; } memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr)); @@ -1258,6 +1257,9 @@ defConstant(MalBlkPtr mb, int type, ValP return k; } k = newTmpVariable(mb, type); + if (k == -1) { + return k; + } setVarConstant(mb, k); setVarFixed(mb, k); if (type >= 0 && type < GDKatomcnt && ATOMextern(type)) diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c --- a/monetdb5/mal/mal_interpreter.c +++ b/monetdb5/mal/mal_interpreter.c @@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size) assert(size >= mb->vsize); stk = newGlobalStack(size); + if (!stk) { + return NULL; + } //memset((char *)stk, 0, stackSize(size)); already set //stk->stksize = size; stk->stktop = mb->vtop; diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c --- a/monetdb5/mal/mal_stack.c +++ b/monetdb5/mal/mal_stack.c @@ -59,8 +59,9 @@ newGlobalStack(int size) MalStkPtr s; s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk)); - if (s == NULL) - GDKfatal("newGlobalStack:can not obtain memory\n"); + if (!s) { + return NULL; + } s->stksize = size; return s; } @@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn return old; k = ((cnt / STACKINCR) + 1) * STACKINCR; s = newGlobalStack(k); + if (!s) { + return NULL; + } memcpy(s, old, stackSize(old->stksize)); s->stksize = k; GDKfree(old); diff --git a/monetdb5/optimizer/opt_mergetable.c b/monetdb5/optimizer/opt_mergetable.c --- a/monetdb5/optimizer/opt_mergetable.c +++ b/monetdb5/optimizer/opt_mergetable.c @@ -1647,8 +1647,9 @@ OPTmergetableImplementation(Client cntxt ml.vsize = mb->vsize; ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize); - if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) + if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) { goto cleanup; + } for (i=0; i 0){ chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE); diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c --- a/monetdb5/optimizer/opt_reorder.c +++ b/monetdb5/optimizer/opt_reorder.c @@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr sz += list[i]->used; } uselist = GDKzalloc(sizeof(int)*sz); + if (!uselist) { + GDKfree(list); + GDKfree(var); + return NULL; + } for(i=0;istop; i++) { if (list[i]->cnt) { diff --git a/sql/backends/monet5/sql_optimizer.c b/sql/backends/monet5/sql_optimizer.c --- a/sql/backends/monet5/sql_optimizer.c +++ b/sql/backends/monet5/sql_optimizer.c @@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu return size; } -static lng +static lng SQLgetSpace(mvc *m, MalBlkPtr mb, int prepare) { sql_trans *tr = m->session->tr; - lng size,space = 0, i; + lng size,space = 0, i; for (i = 0; i < mb->stop; i++) { InstrPtr p = mb->stmt[i]; @@ -80,7 +80,7 @@ SQLgetSpace(mvc *m, MalBlkPtr mb, int pr sql_table *t = 0; sql_column *c = 0; - if (!s || strcmp(s->base.name, dt_schema) == 0) +
MonetDB: Dec2016 - Added test for Bug 6134
Changeset: d21bef4f8fdd for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d21bef4f8fdd Modified Files: sql/test/BugTracker-2017/Tests/All Branch: Dec2016 Log Message: Added test for Bug 6134 diffs (8 lines): diff --git a/sql/test/BugTracker-2017/Tests/All b/sql/test/BugTracker-2017/Tests/All --- a/sql/test/BugTracker-2017/Tests/All +++ b/sql/test/BugTracker-2017/Tests/All @@ -1,3 +1,4 @@ shutdown.Bug-6182 avggroupbysq.Bug-6178 semijoinunion.Bug-6150 +heapextend.Bug-6134 ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - new files, Bug 6134
Changeset: f0eaa26022fe for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f0eaa26022fe Added Files: sql/test/BugTracker-2017/Tests/all_dates.csv.gz sql/test/BugTracker-2017/Tests/heapextend.Bug-6134.sql.in Branch: Dec2016 Log Message: new files, Bug 6134 diffs (truncated from 1704 to 300 lines): diff --git a/sql/test/BugTracker-2017/Tests/all_dates.csv.gz b/sql/test/BugTracker-2017/Tests/all_dates.csv.gz new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5b660b69ded583f9ba5235a6e32aa016a31820e7 GIT binary patch literal 92846 zc%1EBeNau$TFi-oly1ZS~Rk+BYbqBL@Mt`u9TUY8cUBGN5Y>F{-P5yOyZ z#yWG=6q0SHYi$+SrBn14fnpI52(ed_%+@MH{XhvMNT!0Yd_zC_|Iy!VT_oM`@#1;!l)d@((Is17{(8foPdolm zyJ&mrR*ir3djX&I{55~cPHBD5mkoaU@T-f*J{bGj{ek0aW!b8_+SkwY{NXkKEvikX zM6v&xo~Ew?UuJ%=i1~rz>-0BjTguYnO)u3*H<|w9wSZ-+WYgBpBZU8+PZu-Scp6;e zNI-E~UhfgszQjpuzGVKG6!@{z^OhdHnY6bkd}mmqAe<*X)$>(;#CquhQ*lqBmjh`j z6P+>gfJOH2_Oi&A+a5|wB2{}XEqPV-@20n#{c}`tg0Pl^lUtcT77P5?+eeRW7>MXd zlZsHkyBYT>rMvNy(yyYyf37OQ^oQnvP*ss7CoiVV`?;(%hWrgU&r?xe&a<*Q{p9K& zfuF0&ex0#K`qL$BpuD_&US(=o;h`L|M7MlSi;{(y(r}Uk=QIW9Fi(8nJbX+rWM^F+ zRcVZaSnKlg;{eb$3%mg>wJj>k7OmKIa`k~rU*#_@k$%=wX<3|oNtn0dP3BrR0&88; z_DQ;ZUS6=+Y978hZoToQB#8B{E5C)b-iHB|HD9d)P`&iZ<4{fiV-pRwkTj_VU2^+^ zE2857iD-n8h>Uns5V}NkrRAA%Bx1NQd_A&6&?=&hm(Yp)-TE?fw^n(oUUrD6_L&Mj zot77ZA-pedH*`1AD)eTQg=Bp#DYry;Mfd*Lkc7V@t<^5QdFw~;;*Y^-8Cj$%4Ov?x z?Umgq1}-qA3+CcimFE|2Q?bNd>P_4qN{uy*ph~Ma0ig;D)c&Q&ddXop<>5x;??T%| zLn)FiIT^uX32TRv3acN4=n9C9fDw#Vc@z2lv4&a`L96W!iyuoj-GIJ{422`U5IW_8IHTt~RrqF~Kp0`CXMT$)Bl0)1?QP{@i zNc4{W$3R4oStca(BE51tJE-LTk~08&}{txac_|RlgbjwR)0yX z_scw;CIyMc<26odTTt?LK>E9WgwYz2<@p$AbG4>17sm!Ss{N}md;@->P!22-!Y|k# z+j_ib1*!--&^;Kb0;R&c=!O-{ZE4qE)2;$NW#tX~5r~^;5H7-SC~0FaEVxWNPe;&s zX`|*lGbt))qW>f@MbfrHl?rj0{m0*nL(w?>Nww-@Hv#FdUkT%E7l!jN&L;PaF&M|$ zu8LG;V9;RsjV>%2yrSHKEP@Vni{b^KRCpKNBf3L`ou+wmUph|dCs&ijy5*Y}#30kJ z14l~)L;Gp(sl-R#y)y*$uXUg;{e}xc`8EW#^I$1W^Y(x=57vMB#Tz*Olg>o&J`)f) z{!^_cAfHlb1_iT7E(kSayr=Faszf(BRe0&CK8o_AK7Wt2<`!6AWW$a2pF%kQX`1>^ z)PN+G8h_`L*5)st)AB$>(^@WII+FsXSM=AC)u1m@-xvz9*6)wM=i_+kb-~aey2I(xLUbI6KV8H=tP~4y1N56>cHH~0o9V#hWDVViWLI^UrKKoQ2|=x+^}R(@5Is)qb0Mf{9fF!L(>Q7A=vBcGpZ4>vas?)A8!^p&dlAW3 z6dSs&Sea2#98PBRRFULVec?4VPG&SCSX@mj|8XS;Ps!lFdA&Nl=wKy)>^Ye*Hu$B4 z*D!2w@3yYBI5s#XP1=qj6}*ZjEUDNNZwf?}3Wu5)J>CNbX1z<3pCYMyfZ!X0lDgTM zMt%Q*4G=Uq`1hY+B;*=|wkFN4&S;!sy)MJ^VoIUu8dq}gc~3ROW#+}#APMtG#P+mJ zpe;@Q0)kwuQ~S#?&Sq_*5QDGpk8MBR1L}MTjnSy-2mKg08XTA)MixN_y7}b|h@LAg zx*Ld}$uvSbPG%W~Ce~FWHjQ%7viVwsd-X_dk7~)gKt*kQvdkHr{N#-8@{!CGcmK_e85w?Ns5VrDddFOx{Kd#@V}YDf9_V=V=g-A=-r6^)vD|o>6ada(AG%w_@dLyphzz%u>K5X)>`t=faU%(!7*i(RP5 zixyLtm`iQ#Kj5X#*m30@r0jk4lewOi*M)u$Hoq(TELW8SS#=l9TbMsdu2IE!QS7gm zJ&-oORg-+CXR#ahvuR=DJ(jGl)E?Un>@K;(%kGLc+z+VCl4fAwC9@(L3oqy2GIgTj z<$gelwjY$Cs%!|^+nbZnT+d!+v8G zeQ15%d<$DKoiTdL?tGlyQa1RDH|kIYN*t@zo9=6NS)&gipN?dMX7t%mifl*|9b`+D zB*FHg7h<;Pi|W%_|0dNEnTOH+Mjvi`JX8HP>jUr_zk<2Oj`9YKqtVngrU91-BfJ5{ zTK`rIkvLar?7$HT^JE-V^0oe@$jc0E6IpmvRejo2(WHB`4-*fFg6MkA+RrF! zKjEE2DpF!2jv_s0%2n)YT8*Pf+xJ;^(n6I13KgeFr?n!T@XjZ#BF#BW*olHBA1P8t zDr)zFTbV#4;N8bY0>%XR+*^VyqzD52j|Wm&R-Wd!6;Ylr|L?^~^>N;>VqJZB3F6lN zOfNDcx$7JtwKoPK4$_z9?3?~|F|2`Sa%}@z)VV)H_Xr$ zsZUp)pMxraj%DsS(eo|xGQ$ey>U;GgqnU>Z_MQJ{m$yQWFsBFfF?gg9+JaW1y#<%| zW!aD~Vw%LAce7ak@3eznS{ zdXzlU7&2kb;VF4dPGf%%H1@OWIAiZyW1rehS`Ceg3li2+_vCq-4(3qZS+8t7i+(cw zb&a4hgTm)-(LtJvuF<|=dxiOv)QO%_2odfozlEgNMk2n*+5{fxQW%%w=4ut_ia?yo zy1E-7Sa|6y&quQjPDN3bl?TcYlzY}#AFe~ToQ`GQF1MiTvm?5TLfUG68`6RWPe~jG zJUgn2N(Yu|&zT~Sy~(ic+B;43db!BcOk$!UOO>Qu+W|7WZyUW8YdVxgoiYqPA{&s^ zu5KMUM$=J@e0_-aj>y)elMN2pUu1qwy=B@0VP>)u{*GjO9|l~h`wHCRF_>%2IMD-M zKg7|5X!~VYG40|A(l^1SRNS8+Cy-B`V7&*YKf%t9LS!P;p9>8qI^9|3Km>FUcV{JZr7~@w%GL} z#C4=+CG5aBnzZDu8l0H=)huZ*hJV(PCc3#&7@G#rM+Y2+@=|$NGHDYNgEv96D=;Kg=(Ci$0^cjtw<-l^GT{mmu{y- zbM+%is;Zu1gN<6FQgSK#X`8Yyoo}Hul+{48-MmR#MtXZ^x6Nue%W={frAqG;1((6{ zcDSmNPqU;xjzHx_i>*Y#fRC#2X)UV#5!9laI4$bq!l$7@bBk@hS#&K7f4~(%Xop*bB$;(ka8G(VvO=y zthCjfRs~?()aEwq}l_&%^2Q@;?e7pw~C8+nF zJVPfCM9IMgS|#n;wT{#yQ!jasOF74wl4CQ?P~;+Mr86#NpuH(%g%P*6=%V?dP?2I7 zvcJgu8X~~Yn!}UqAuo`F_dHuf9K45w8eWAKSRmZ^rnTkyGIVWGRnrJ=4(XcwBPyCL%V4!vZ0G!Po{&fjrOIC13j1KZWrtX;xPn~5nC z1-F=$1B1N71e=_A&*!rq&Vx2m4wSwNUgUF7x{X8WKM$oZh$*YVXnK~yL8;(F9F)>z z)Fud&XmC)vr};uFO`!up3jLH&%6Q%}+2i^nJ@1Hwn$c}tAlHMMyLUBGm5XlE_7>ep z!4e66dA=NVbF60}Q8>_}Zl^-pm1Z&ph;Aq6+&$&ka7ZNIpvYOJ->~`sG8wY}c92pW zCP+QaP$cXhhDbxPrb9wXGIKhln?@l{heQ%!eqwzbjwh2FZwf}gij=gxc3^XGuk^{o zb;z4zO_f9u-6IF)=&4Z3)#5f4Na?2n%U%7*F_51?)R?j+>qA;8#$IhgH}R5kbjW6#9XTP zM5$ACM?D&Bq?&Z297bmpMvb(CJTBS@nF^J};iN*H z1yO=5oK)yK%^eZhq`T@PcZo~bPmW}0%Y|NLj4^nsBs=hG8kc4d3LfiOr^j7`IJ~>FCy$>vN9Mbhj6;d|F@mAi_ zN??oGA0WMzhlCo@nQSmN4&}%69`9L;t}PnX{Z&{Z!AodH775lHdnne$F`v{iN^#0m zm6Ql9>uCzThpQ@KS+wcy+B>?PR!otqlH|Q4!56nucA>&PpM56~Q}u9A%6fOy{bPvl zj&i#6Np-2c`2rO~>O;+`XK>v6lM8X&ds|41WQSUhE(`NC*EuIxpY0Ko~BB=UU#vcCU`Pm#_{vcRAFce z$%c$zXDi7#Y2x5Ho#E+x4r$6b^^tS*2HL3~SEKcGoO);IDe@aII&+WqDUW|6{EDNwRH^BxD{CtFmv1zW{!2!#)Nxmp7`al%I_jY-l4UoF-7mW~ z1i*V!tIG2|Px>Yzms@)_hn6cv*c>}oN^8nagj}if;heExBn=2b1WrDmmhEh;e7s6ou~u(2eB8xi^QH&qvrCE1!=dx{iFlSs7#G
MonetDB: queryid - merge with default
Changeset: 312f53b33312 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=312f53b33312 Added Files: README.rst clients/Tests/exports.stable.err-noapprove clients/Tests/malcheck.stable.err-noapprove clients/Tests/malcheck.stable.out-noapprove debian/monetdb-python2.install monetdb5/modules/kernel/Tests/math.malC monetdb5/modules/kernel/Tests/math.stable.err monetdb5/modules/kernel/Tests/math.stable.out monetdb5/modules/mal/oltp.c monetdb5/modules/mal/oltp.h monetdb5/modules/mal/oltp.mal monetdb5/optimizer/opt_oltp.c monetdb5/optimizer/opt_oltp.h monetdb5/tools/Tests/mserver5--help.stable.err.Windows sql/backends/monet5/Tests/createorreplace.sql sql/backends/monet5/Tests/createorreplace.stable.err sql/backends/monet5/Tests/createorreplace.stable.out sql/backends/monet5/Tests/pyapi33.sql sql/backends/monet5/Tests/pyapi33.stable.err sql/backends/monet5/Tests/pyapi33.stable.out sql/backends/monet5/sql_cast_impl_int.h sql/backends/monet5/sql_cat.c sql/backends/monet5/sql_cat.h sql/backends/monet5/sql_transaction.c sql/backends/monet5/sql_transaction.h sql/backends/monet5/sql_transaction.mal sql/backends/monet5/sqlcatalog.mal sql/test/BugTracker-2015/Tests/long_insert.Bug-3845.timeout sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.sql sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.err sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.out sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.out.int128 sql/test/BugTracker-2017/Tests/All sql/test/BugTracker-2017/Tests/all_dates.csv.gz sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.sql sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.err sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.sql sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.stable.err sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.stable.out sql/test/BugTracker-2017/Tests/heapextend.Bug-6134.sql.in sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.sql sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.stable.err sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.stable.out sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.sql sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.stable.err sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.stable.out sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.sql sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.stable.err sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.stable.out sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.sql sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.err sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.out sql/test/Tests/bincopycollist-prepare.py sql/test/Tests/ifexists.sql sql/test/Tests/ifexists.stable.err sql/test/Tests/ifexists.stable.out sql/test/Tests/subqueries.sql sql/test/Tests/subqueries.stable.err sql/test/Tests/subqueries.stable.out sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out-noapprove sql/test/emptydb-upgrade-chain-hge/Tests/dump.stable.err-noapprove sql/test/emptydb-upgrade-chain-hge/Tests/dump.stable.out-noapprove sql/test/emptydb-upgrade-chain/Tests/check.stable.out-noapprove sql/test/emptydb-upgrade-chain/Tests/dump.stable.err-noapprove sql/test/emptydb-upgrade-chain/Tests/dump.stable.out-noapprove sql/test/emptydb-upgrade-hge/Tests/check.stable.out-noapprove sql/test/emptydb-upgrade-hge/Tests/dump.stable.err-noapprove sql/test/emptydb-upgrade-hge/Tests/dump.stable.out-noapprove sql/test/emptydb-upgrade/Tests/check.stable.out-noapprove sql/test/emptydb-upgrade/Tests/dump.stable.err-noapprove sql/test/emptydb-upgrade/Tests/dump.stable.out-noapprove sql/test/emptydb/Tests/dump.stable.err-noapprove sql/test/emptydb/Tests/dump.stable.out-noapprove sql/test/malloc_fail/Tests/mallocs.SQL.py sql/test/sys-schema/Tests/All sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.err sql/test/sys-schema/Tests/check_AlternateKey
MonetDB: queryid - closing branch dude
Changeset: 614d7bbd027b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=614d7bbd027b Branch: queryid Log Message: closing branch dude ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: queryid2 - queryid re-implementation
Changeset: 6774196d9fba for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6774196d9fba Modified Files: monetdb5/mal/mal_runtime.c sql/backends/monet5/sql.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_result.h sql/include/sql_catalog.h sql/storage/bat/res_table.c sql/storage/sql_storage.h Branch: queryid2 Log Message: queryid re-implementation diffs (287 lines): diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c --- a/monetdb5/mal/mal_runtime.c +++ b/monetdb5/mal/mal_runtime.c @@ -82,6 +82,7 @@ runtimeProfileInit(Client cntxt, MalBlkP // add new invokation QRYqueue[i].mb = mb; QRYqueue[i].tag = qtag++; + mb->tag = QRYqueue[i].tag; QRYqueue[i].stk = stk; // for status pause 'p'/running '0'/ quiting 'q' QRYqueue[i].start = (lng)time(0); QRYqueue[i].runtime = mb->runtime; // the estimated execution time diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -1705,7 +1705,7 @@ mvc_result_set_wrap( Client cntxt, MalBl b = BATdescriptor(bid); if ( b == NULL) throw(MAL,"sql.resultset","Failed to access order column"); - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, b); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 5), 1, b); if (res < 0) msg = createException(SQL, "sql.resultSet", "failed"); BBPunfix(b->batCacheid); @@ -1796,7 +1796,7 @@ mvc_export_table_wrap( Client cntxt, Mal order = BATdescriptor(bid); if ( order == NULL) throw(MAL,"sql.resultset","Failed to access order column"); - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, order); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 11), 1, order); t = m->results; if (res < 0){ msg = createException(SQL, "sql.resultSet", "failed"); @@ -1895,7 +1895,7 @@ mvc_row_result_wrap( Client cntxt, MalBl return msg; if ((msg = checkSQLContext(cntxt)) != NULL) return msg; - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, NULL); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 5), 1, NULL); tbl = BATdescriptor(tblId); atr = BATdescriptor(atrId); @@ -1970,7 +1970,7 @@ mvc_export_row_wrap( Client cntxt, MalBl return msg; if ((msg = checkSQLContext(cntxt)) != NULL) return msg; - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, NULL); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 11), 1, NULL); t = m->results; if (res < 0){ @@ -2071,7 +2071,7 @@ mvc_table_result_wrap(Client cntxt, MalB if ((order = BATdescriptor(*order_bid)) == NULL) { throw(SQL, "sql.resultSet", "Cannot access descriptor"); } - *res_id = mvc_result_table(m, *nr_cols, *qtype, order); + *res_id = mvc_result_table(m, mb->tag, *nr_cols, *qtype, order); if (*res_id < 0) res = createException(SQL, "sql.resultSet", "failed"); BBPunfix(order->batCacheid); @@ -2203,7 +2203,7 @@ mvc_affected_rows_wrap(Client cntxt, Mal assert(mtype == TYPE_lng); nr = *getArgReference_lng(stk, pci, 2); b = cntxt->sqlcontext; - error = mvc_export_affrows(b, b->out, nr, ""); + error = mvc_export_affrows(b, b->out, nr, "", mb->tag); if (error) throw(SQL, "sql.affectedRows", "failed"); return MAL_SUCCEED; @@ -2301,12 +2301,11 @@ mvc_scalar_value_wrap(Client cntxt, MalB str *type = getArgReference_str(stk, pci, 3); int *digits = getArgReference_int(stk, pci, 4); int *scale = getArgReference_int(stk, pci, 5); - int *eclass = getArgReference_int(stk, pci, 6); ptr p = getArgReference(stk, pci, 7); int mtype = getArgType(mb, pci, 7); str msg; backend *b = NULL; - + int res_id; (void) mb; /* NOT USED */ if ((msg = checkSQLContext(cntxt)) != NULL) return msg; @@ -2315,13 +2314,14 @@ mvc_scalar_value_wrap(Client cntxt, MalB p = *(ptr *) p; // scalar values are single-column result sets - mvc_result_table(b->mvc, 1, 1, NULL); + res_id = mvc_result_table(b->mvc, mb->tag, 1, 1, NULL); mvc_result_value(b->mvc, *tn, *cn, *type, *digits, *scale, p, mtype); if (b->output_format == OFMT_NONE) { return MAL_SUCCEED; } - if (b->out == NULL || mvc_export_value(b, b->out, 1, *tn, *cn, *type, *digits, *scale, *eclass, p, mtype, "", "NULL")
MonetDB: queryid2 - approve test output
Changeset: 6939acb0621d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6939acb0621d Modified Files: sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out Branch: queryid2 Log Message: approve test output diffs (70 lines): diff --git a/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out b/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out --- a/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out +++ b/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out @@ -25,40 +25,40 @@ Ready. # 11:50:59 > #select CASE WHEN 1=1 THEN 1 ELSE NULL END; -% .L1 # table_name -% L1 # name +% .L2 # table_name +% L2 # name % tinyint # type % 1 # length [ 1] #select CASE WHEN 1=0 THEN 1 ELSE NULL END; -% .L1 # table_name -% L1 # name +% .L2 # table_name +% L2 # name % tinyint # type -% 4 # length +% 1 # length [ NULL ] #select CASE WHEN 2=2 THEN 2 else cast(null as char) end; -% .L2 # table_name -% L2 # name +% .L3 # table_name +% L3 # name % char # type % 1 # length [ "2" ] #select CASE WHEN 2=0 THEN 2 else cast(null as char) end; -% .L2 # table_name -% L2 # name +% .L3 # table_name +% L3 # name % char # type % 1 # length [ NULL ] #select CASE WHEN 3=3 THEN 3 else cast(null as int) end; -% .L2 # table_name -% L2 # name +% .L3 # table_name +% L3 # name % int # type % 1 # length [ 3] #select CASE WHEN 3=0 THEN 3 else cast(null as int) end; -% .L2 # table_name -% L2 # name +% .L3 # table_name +% L3 # name % int # type -% 11 # length +% 1 # length [ NULL ] # 11:35:46 > diff --git a/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out b/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out --- a/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out +++ b/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out @@ -56,7 +56,7 @@ Ready. % .L2 # table_name % L2 # name % int # type -% 11 # length +% 1 # length [ NULL ] # 18:33:32 > ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: queryid2 - merge with default
Changeset: c224e8a2b82a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c224e8a2b82a Added Files: monetdb5/extras/rapi/Tests/rapi02.stable.out.int128 sql/backends/monet5/Tests/int_notation_1e5.stable.out.int128 sql/backends/monet5/UDF/pyapi/Tests/pyapi_types_huge.stable.out.int128 sql/jdbc/tests/Tests/Test_Int128.stable.out.int128 sql/test/BugTracker-2015/Tests/hugeint-wrong-value.Bug-3849.stable.out.int128 sql/test/BugTracker-2016/Tests/convert-function-test-hge.Bug-3460.stable.out.int128 sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out.int128 sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.sql sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.stable.err sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.stable.out sql/test/BugTracker-2017/Tests/dce_bug-6177.sql sql/test/BugTracker-2017/Tests/dce_bug-6177.stable.err sql/test/BugTracker-2017/Tests/dce_bug-6177.stable.out sql/test/BugTracker-2017/Tests/group-by-on-constant.Bug-6082.stable.out.int128 sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.sql sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.stable.err sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.stable.out sql/test/BugTracker-2017/Tests/sqlsmith01.sql sql/test/BugTracker-2017/Tests/sqlsmith01.stable.out sql/test/BugTracker-2017/Tests/sqlsmith02.sql sql/test/BugTracker-2017/Tests/sqlsmith02.stable.out sql/test/BugTracker-2017/Tests/sqlsmith03.sql sql/test/BugTracker-2017/Tests/sqlsmith03.stable.out sql/test/BugTracker-2017/Tests/sqlsmith04.sql sql/test/BugTracker-2017/Tests/sqlsmith04.stable.out sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.sql sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.stable.err sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.stable.out sql/test/emptydb/Tests/check.SQL.py sql/test/mapi/Tests/mal_int128.stable.out.int128 sql/test/mapi/Tests/perl_dec38.stable.out.int128 sql/test/mapi/Tests/perl_int128.stable.out.int128 sql/test/mapi/Tests/php_dec38.stable.out.int128 sql/test/mapi/Tests/php_int128.stable.out.int128 sql/test/mapi/Tests/python2_dec38.stable.out.int128 sql/test/mapi/Tests/python2_int128.stable.out.int128 sql/test/mapi/Tests/python3_dec38.stable.out.int128 sql/test/mapi/Tests/python3_int128.stable.out.int128 sql/test/mapi/Tests/sql_dec38.stable.out.int128 sql/test/mapi/Tests/sql_int128.stable.out.int128 sql/test/mergetables/Tests/addtable.sql sql/test/mergetables/Tests/addtable.stable.err sql/test/mergetables/Tests/addtable.stable.out sql/test/pg_regress/Tests/numeric.stable.out.int128 sql/test/testdb-upgrade-chain-hge/Tests/dump.stable.out.int128 sql/test/testdb-upgrade-chain-hge/Tests/package.stable.out.int128 sql/test/testdb-upgrade-chain-hge/Tests/unpackage.stable.out.int128 sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/dump.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/unpackage.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/testdb/Tests/testdb-package-hge.stable.out.int128 Removed Files: clients/Tests/SQL-dump.SQL.py clients/Tests/SQL-dump.stable.err clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.int128 gdk/bat.feps gdk/bat.png gdk/bat1.feps gdk/bat1.png gdk/bat2.feps gdk/bat2.png monetdb5/extras/rapi/Tests/rapi02.stable.out sql/backends/monet5/Tests/int_notation_1e5.stable.out sql/backends/monet5/UDF/pyapi/Tests/pyapi_types_huge.stable.out sql/jdbc/tests/Tests/Test_Int128.stable.out sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.err.single sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.out.single sql/test/BugTracker-2015/Tests/hugeint-wrong-value.Bug-3849.stable.out sql/test/BugTracker-2015/Tests/useless_casts.Bug-3756.stable.out.single sql/test/BugTracker-2016/Tests/convert-function-test-hge.Bug-3460.stable.out sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit sql/test/emptydb/Tests/check.SQL.py.src sql/test/mapi/Tests/mal_int128.stable.out sql/test/mapi/Tests/perl_dec38.stable.out sql/test/mapi/Tests/perl_int128.stable.out sql/test/mapi/Tests/php_dec38.stable.out sql/test/mapi/Tests/ph
MonetDB: queryid2 - merge with default
Changeset: 0210c5d6072b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=3Dchangeset;node=3D0210c5d6072b Modified Files: MonetDB.spec geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err monetdb5/modules/mal/mkey.c sql/backends/monet5/sql.c sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.stable= .err sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stabl= e.err sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.e= rr sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err sql/test/BugTracker-2015/Tests/backslash_predecing_separator.Bug-3775.stab= le.err sql/test/BugTracker-2017/Tests/All sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err sql/test/copy/Tests/columns.stable.err sql/test/copy/Tests/int_parse.stable.err sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err sql/test/copy/Tests/null_as_string_errors.stable.err Branch: queryid2 Log Message: merge with default diffs (truncated from 315 to 300 lines): diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -201,6 +201,7 @@ package, and most likely also %{name}-SQ more client packages. =20 %files +%license COPYING %defattr(-,root,root) %{_libdir}/libbat.so.* =20 @@ -242,6 +243,7 @@ This package contains a shared library ( various other components. =20 %files stream +%license COPYING %defattr(-,root,root) %{_libdir}/libstream.so.* =20 @@ -289,6 +291,7 @@ SQL database so that it can be loaded ba MonetDB, you will very likely need this package. =20 %files client +%license COPYING %defattr(-,root,root) %{_bindir}/mclient %{_bindir}/msqldump @@ -375,6 +378,7 @@ odbcinst -u -d -n MonetDB fi =20 %files client-odbc +%license COPYING %defattr(-,root,root) %{_libdir}/libMonetODBC.so %{_libdir}/libMonetODBCs.so @@ -853,6 +857,7 @@ MonetDB packages. You probably don't ne developer. If you do want to test, install %{name}-testing-python. =20 %files testing +%license COPYING %defattr(-,root,root) %{_bindir}/Mdiff %{_bindir}/MkillUsers diff --git a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err b/= geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err --- a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err +++ b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err @@ -13,7 +13,7 @@ stderr of test 'copy_into_crash.SF-19754 MAPI =3D (monetdb) /var/tmp/mtest-27003/.s.monetdb.30235 QUERY =3D COPY 1 RECORDS INTO nodes from STDIN USING DELIMITERS ',', '\n'; 45111956, 'POINT(52.0697 4.3723)' -ERROR =3D !Failed to import table line 1 field p 'geometry(4)' expected in= ' 'POINT(52.0697 4.3723)'' +ERROR =3D !Failed to import table 'nodes', line 1 field p 'geometry(4)' ex= pected in ' 'POINT(52.0697 4.3723)'' =20 # 13:33:18 > =20 # 13:33:18 > Done. diff --git a/monetdb5/modules/mal/mkey.c b/monetdb5/modules/mal/mkey.c --- a/monetdb5/modules/mal/mkey.c +++ b/monetdb5/modules/mal/mkey.c @@ -50,6 +50,11 @@ MKEYhash(Client cntxt, MalBlkPtr mb, Mal res=3D getArgReference_lng(stk,p,0); val=3D getArgReference(stk,p,1); switch (ATOMstorage(tpe)) { + case TYPE_void: + case TYPE_bat: + case TYPE_ptr: + // illegal types, avoid falling into the default case. + assert(0); case TYPE_bte: *res =3D MKEYHASH_bte(val); break; diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -2507,7 +2507,7 @@ mvc_import_table_wrap(Client cntxt, MalB if (fname && s =3D=3D NULL) throw(IO, "bstreams.create", "Failed to create block stream"); if (b =3D=3D NULL) - throw(SQL, "importTable", "Failed to import table %s", be->mvc->errstr); + throw(SQL, "importTable", "Failed to import table '%s', %s", t->base.nam= e, be->mvc->errstr); bat2return(stk, pci, b); GDKfree(b); return msg; diff --git a/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-11= 00504.stable.err b/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into= .SF-1100504.stable.err --- a/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.s= table.err +++ b/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.s= table.err @@ -88,9 +88,7 @@ QUERY =3D COPY 1 RECORDS INTO "news" FROM=20 allemaal nogal lang duurt, maar in principe is het nog maar een uurtje of 6 werk om het goed te krijgen. Zoals
MonetDB: queryid2 - merge with default
Changeset: 3de277fe54c3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=3Dchangeset;node=3D3de277fe54c3 Removed Files: sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out= .32bit Modified Files: monetdb5/mal/mal_function.c monetdb5/mal/mal_profiler.c monetdb5/tools/Tests/mserver5--help.bat monetdb5/tools/Tests/mserver5--help.stable.err.Windows sql/backends/monet5/UDF/pyapi/convert_loops.h sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Darwin sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Windows sql/test/BugTracker-2017/Tests/All sql/test/BugTracker-2017/Tests/sqlsmith02.stable.out sql/test/BugTracker-2017/Tests/sqlsmith03.stable.out Branch: queryid2 Log Message: merge with default diffs (truncated from 414 to 300 lines): diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c --- a/monetdb5/mal/mal_function.c +++ b/monetdb5/mal/mal_function.c @@ -924,7 +924,7 @@ showFlowDetails(MalBlkPtr mb, MalStkPtr=20 { (void) mb; /* fool the compiler */ (void) stk; /* fool the compiler */ - mnstr_printf(f, "n%d [fontsize=3D8, shape=3Dbox, label=3D\"%s\"]\n", pc, = getFunctionId(p)); + mnstr_printf(f, "n%d [fontsize=3D8, shape=3Dbox, label=3D\"%s\"]\n", pc, = getFunctionId(p) ? getFunctionId(p) : ""); } =20 /* Produce a file with the flow graph in dot format. diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c --- a/monetdb5/mal/mal_profiler.c +++ b/monetdb5/mal/mal_profiler.c @@ -125,7 +125,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk if( usrname) logadd("\"user\":\"%s\",%s",usrname, prettify); logadd("\"clk\":"LLFMT",%s",usec,prettify); - logadd("\"ctime\":"LLFMT".%06ld\",%s", clock / 100, (long) (clock % 1= 00), prettify); + logadd("\"ctime\":"LLFMT".%06ld,%s", clock / 100, (long) (clock % 100= ), prettify); logadd("\"thread\":%d,%s", THRgettid(),prettify); =20 logadd("\"function\":\"%s.%s\",%s", getModuleId(getInstrPtr(mb, 0)), getF= unctionId(getInstrPtr(mb, 0)), prettify); @@ -400,7 +400,7 @@ profilerHeartbeatEvent(char *alter) lognew(); logadd("{%s",prettify); // fill in later with the event counter logadd("\"user\":\"heartbeat\",%s", prettify); - logadd("\"ctime\":"LLFMT".%06ld\",%s", clock / 100, (long) (clock % 1= 00), prettify); + logadd("\"ctime\":"LLFMT".%06ld,%s", clock / 100, (long) (clock % 100= ), prettify); logadd("\"rss\":"SZFMT ",%s", MT_getrss()/1024/1024, prettify); #ifdef HAVE_SYS_RESOURCE_H getrusage(RUSAGE_SELF, &infoUsage); diff --git a/monetdb5/tools/Tests/mserver5--help.bat b/monetdb5/tools/Tests= /mserver5--help.bat --- a/monetdb5/tools/Tests/mserver5--help.bat +++ b/monetdb5/tools/Tests/mserver5--help.bat @@ -1,4 +1,4 @@ @echo on @prompt # $t $g =20 =20 - %MTIMEOUT% mserver5 --help + %MTIMEOUT% mserver5.exe --help diff --git a/monetdb5/tools/Tests/mserver5--help.stable.err.Windows b/monet= db5/tools/Tests/mserver5--help.stable.err.Windows --- a/monetdb5/tools/Tests/mserver5--help.stable.err.Windows +++ b/monetdb5/tools/Tests/mserver5--help.stable.err.Windows @@ -5,11 +5,6 @@ stderr of test 'mserver5--help` in direc # 16:36:12 > Mtimeout -timeout 300 ./mserver5--help mserver5--help=20 # 16:36:12 > =20 =20 - -# 16:36:12 > =20 -# 16:36:12 > Mtimeout -timeout 60 mserver5 --config=3D --help -# 16:36:12 > =20 - Usage: mserver5.exe [options] [scripts] --dbpath=3D Specify database location --dbextra=3D Directory for transient BATs diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h b/sql/backends/m= onet5/UDF/pyapi/convert_loops.h --- a/sql/backends/monet5/UDF/pyapi/convert_loops.h +++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h @@ -24,6 +24,37 @@ =20 // This #define creates a new BAT with the internal data and mask from a N= umpy array, without copying the data // 'bat' is a BAT* pointer, which will contain the new BAT. TYPE_'mtpe' is= the BAT type, and 'batstore' is the heap storage type of the BAT (this sho= uld be STORE_CMEM or STORE_SHARED) +#if defined(_MSC_VER) && _MSC_VER <=3D 1600 +#define isnan(x) _isnan(x) +#endif + +#define nancheck_flt(bat) \ + do {\ + for (iu =3D 0; iu < ret->count; iu++) { \ + if (isnan(((flt*)data)[index_offset * ret->count + iu])) { \ + ((flt*)data)[index_offset * ret->count + iu] =3D flt_nil; \ + bat->tnil =3D 1; \ + } \ + } \ +
MonetDB: queryid2 - approved test output
Changeset: 72179ed3f8bd for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=72179ed3f8bd Modified Files: sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out sql/test/Tests/identifiers.stable.out sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out sql/test/VOC/Tests/median.Bug-3096.stable.out sql/test/pg_regress/Tests/date.stable.out sql/test/pg_regress/Tests/strings.stable.out sql/test/pg_regress/Tests/strings_concat.stable.out Branch: queryid2 Log Message: approved test output diffs (truncated from 354 to 300 lines): diff --git a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out --- a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out +++ b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out @@ -36,7 +36,7 @@ Ready. % .L12 # table_name % suma # name % bigint # type -% 20 # length +% 1 # length [ NULL ] #SELECT SUM(a) AS suma #FROM ( diff --git a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 --- a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 +++ b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 @@ -36,7 +36,7 @@ Ready. % .L12 # table_name % suma # name % hugeint # type -% 40 # length +% 1 # length [ NULL ] #SELECT SUM(a) AS suma #FROM ( diff --git a/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out b/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out --- a/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out +++ b/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out @@ -34,7 +34,7 @@ Ready. % .L2 # table_name % L2 # name % int # type -% 11 # length +% 1 # length [ NULL ] #create function call_function( aa int ) RETURNS int #begin @@ -46,7 +46,7 @@ Ready. % .L2 # table_name % L2 # name % int # type -% 11 # length +% 1 # length [ NULL ] # 22:37:37 > diff --git a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out --- a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out +++ b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out @@ -37,7 +37,7 @@ Ready. [ "€", 1 ] #select 'Liever €uro' as "Liever euro"; % .L2 # table_name -% Liever euro # name +% "Liever euro" # name % char # type % 11 # length [ "Liever €uro"] diff --git a/sql/test/Tests/identifiers.stable.out b/sql/test/Tests/identifiers.stable.out --- a/sql/test/Tests/identifiers.stable.out +++ b/sql/test/Tests/identifiers.stable.out @@ -58,13 +58,13 @@ Ready. #CREATE TABLE "B\"la\"" (id int); -- should fail #SELECT 1 AS "B\"la\""; -- should fail % .L2 # table_name -% B\"la\" # name +% "B\\\"la\\\"" # name % tinyint # type % 1 # length [ 1] #SELECT 1 AS "\"Bla\""; -- should fail % .L2 # table_name -% \"Bla\" # name +% "\\\"Bla\\\"" # name % tinyint # type % 1 # length [ 1] diff --git a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py --- a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py +++ b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py @@ -19,7 +19,7 @@ def query(conn, sql): try: cur.execute(sql) except pymonetdb.OperationalError, e: -print e +print "!", e return r = cur.fetchall() cur.close() @@ -30,9 +30,10 @@ def run(conn, sql): try: r = conn.execute(sql) except pymonetdb.OperationalError, e: -print e +print "!", e return -print(r) +print '# OK' + c1 = connect('monetdb', 'monetdb') # Create a new schema with a new table. diff --git a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out --- a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out +++ b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out @@ -59,57 +59,43 @@ Ready. # 10:44:34 > CREATE SCHEMA new_schema_as_well -&3 - +# OK SET SCHEMA new_schema_as_well -&4 t - +# OK CREATE TABLE test (x int, y int) -&3 - +# OK INSERT INTO test VALUES (-1, -1) -&2 1 -1 - +# OK CREATE USER new_use
MonetDB: queryid2 - merge with default. PAAAANOS :)
Changeset: d9d7507d0a5f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d9d7507d0a5f Added Files: sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.sql sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.stable.err sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.stable.out sql/test/BugTracker-2017/Tests/prepare.Bug-6133.sql sql/test/BugTracker-2017/Tests/prepare.Bug-6133.stable.err sql/test/BugTracker-2017/Tests/prepare.Bug-6133.stable.out sql/test/BugTracker-2017/Tests/sqlsmith01.stable.err sql/test/BugTracker-2017/Tests/sqlsmith02.stable.err sql/test/BugTracker-2017/Tests/sqlsmith03.stable.err sql/test/BugTracker-2017/Tests/sqlsmith04.stable.err Modified Files: gdk/gdk.h gdk/gdk_batop.c gdk/gdk_join.c gdk/gdk_orderidx.c gdk/gdk_private.h gdk/gdk_search.c gdk/gdk_select.c monetdb5/mal/mal_profiler.c monetdb5/mal/mal_runtime.c monetdb5/modules/mal/orderidx.c sql/server/sql_parser.y sql/storage/bat/bat_storage.c sql/test/BugTracker-2016/Tests/All sql/test/BugTracker-2017/Tests/All sql/test/BugTracker-2017/Tests/sqlsmith01.stable.out sql/test/BugTracker-2017/Tests/sqlsmith04.stable.out sql/test/emptydb/updatetests sql/test/lateral/Tests/lateral.sql Branch: queryid2 Log Message: merge with default. PNOS :) diffs (truncated from 2109 to 300 lines): diff --git a/gdk/gdk.h b/gdk/gdk.h --- a/gdk/gdk.h +++ b/gdk/gdk.h @@ -587,8 +587,6 @@ typedef size_t BUN; #else #define BUN_NONE ((BUN) LLONG_MAX) #endif -#define BUN_MSK (~BUN_NONE) -#define BUN_UNMSK BUN_NONE #define BUN_MAX (BUN_NONE - 1) /* maximum allowed size of a BAT */ #define BUN2 2 diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c --- a/gdk/gdk_batop.c +++ b/gdk/gdk_batop.c @@ -1729,6 +1729,8 @@ BATcount_no_nil(BAT *b) return cnt; } +/* create a new, dense candidate list with values from `first' up to, + * but not including, `last' */ static BAT * newdensecand(oid first, oid last) { @@ -1738,7 +1740,7 @@ newdensecand(oid first, oid last) return NULL; if (last < first) first = last = 0; /* empty range */ - BATsetcount(bn, last - first + 1); + BATsetcount(bn, last - first); BATtseqbase(bn, first); return bn; } @@ -1790,19 +1792,19 @@ BATmergecand(BAT *a, BAT *b) if (af <= bf && bf <= al + 1) { /* partial overlap starting with a, or b is * smack bang after a */ - return newdensecand(af, al < bl ? bl : al); + return newdensecand(af, al < bl ? bl + 1 : al + 1); } if (bf <= af && af <= bl + 1) { /* partial overlap starting with b, or a is * smack bang after b */ - return newdensecand(bf, al < bl ? bl : al); + return newdensecand(bf, al < bl ? bl + 1 : al + 1); } } if (ad && af <= bf && al >= bl) { - return newdensecand(af, al); + return newdensecand(af, al + 1); } if (bd && bf <= af && bl >= al) { - return newdensecand(bf, bl); + return newdensecand(bf, bl + 1); } bn = COLnew(0, TYPE_oid, BATcount(a) + BATcount(b), TRANSIENT); @@ -1912,7 +1914,7 @@ BATintersectcand(BAT *a, BAT *b) if ((af + BATcount(a) - 1 == al) && (bf + BATcount(b) - 1 == bl)) { /* both lists are VOID */ - return newdensecand(MAX(af, bf), MIN(al, bl)); + return newdensecand(MAX(af, bf), MIN(al, bl) + 1); } bn = COLnew(0, TYPE_oid, MIN(BATcount(a), BATcount(b)), TRANSIENT); diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c --- a/gdk/gdk_join.c +++ b/gdk/gdk_join.c @@ -182,251 +182,6 @@ joininitresults(BAT **r1p, BAT **r2p, BU s##vals + ((x) * s##width)) #define FVALUE(s, x) (s##vals + ((x) * s##width)) -#define BINSEARCHFUNC(TYPE)\ -static inline BUN \ -binsearch_##TYPE(const oid *rcand, oid offset, const TYPE *rvals, \ -BUN lo, BUN hi, TYPE v, int ordering, int last)\ -{ \ - BUN mid;\ - TYPE x; \ - \ - assert(ordering == 1 || ordering == -1);\ - assert(lo <= hi); \ - \ -
MonetDB: queryid2 - more test output approved
Changeset: 3a0b19bb5587 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3a0b19bb5587 Modified Files: sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out.int128 sql/test/bugs/Tests/aggr_single_value.stable.out sql/test/bugs/Tests/aggr_single_value.stable.out.int128 sql/test/pg_regress/Tests/interval.stable.out sql/test/pg_regress/Tests/interval.stable.out.int128 Branch: queryid2 Log Message: more test output approved diffs (truncated from 303 to 300 lines): diff --git a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py --- a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py +++ b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py @@ -1,10 +1,28 @@ import os -from pymonetdb import mapi +import pymonetdb -def open(conn): - conn.connect(database=os.getenv('TSTDB'),username="monetdb",password="monetdb",language="sql",hostname=os.getenv('MAPIHOST'),port=int(os.getenv('MAPIPORT'))) +def connect(): +return pymonetdb.connect(database = os.getenv('TSTDB'), + hostname = '127.0.0.1', + port = int(os.getenv('MAPIPORT')), + username = 'monetdb', + password = 'monetdb', + autocommit = True) -works = """-- some comment +def query(conn, sql, result=False): +cur = conn.cursor() +cur.execute(sql) +r = False +if (result): +r = cur.fetchall() +cur.close() +return r + + +# def open(conn): +# conn.connect(database=os.getenv('TSTDB'),username="monetdb",password="monetdb",language="sql",hostname=os.getenv('MAPIHOST'),port=int(os.getenv('MAPIPORT'))) + +create1 = """-- some comment START TRANSACTION; CREATE TABLE a(i integer); COPY 1 RECORDS INTO a FROM STDIN USING DELIMITERS ',','\\n','"'; @@ -12,7 +30,7 @@ 42 COMMIT; """ -doesnotwork = """-- some comment +create2 = """-- some comment START TRANSACTION; CREATE TABLE b(i integer); COPY 1 RECORDS INTO b FROM STDIN USING DELIMITERS ',','\\n','"'; @@ -24,19 +42,21 @@ 84 COMMIT; """ -cn = mapi.Connection() +cn = connect() +query(cn, create1); +cn.close() -open(cn) -print(cn.cmd('s'+works+';\n')) -cn.disconnect() +cn = connect() +query(cn, create2); +cn.close() -open(cn) -print(cn.cmd('s'+doesnotwork+';\n')) -cn.disconnect() -# this should show a-e, but only shows a -open(cn) -print(cn.cmd('sSELECT name FROM tables WHERE system=0;\n')) -print(cn.cmd('sDROP TABLE a;\n')) +# this should show a-c, but only shows a +cn = connect() +#print() -cn.disconnect() +for x in query(cn, 'SELECT name FROM tables WHERE system=0', True): +print x[0] +query(cn, 'DROP TABLE a') + +cn.close() diff --git a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out --- a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out +++ b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out @@ -17,6 +17,8 @@ stdout of test 'copy-into.Bug-3481` in d # Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-26163/.s.monetdb.33748 # MonetDB/SQL module loaded + + Ready. # SQL catalog created, loading sql scripts once # loading sql script: 09_like.sql @@ -44,35 +46,15 @@ Ready. # loading sql script: 80_udf.sql # loading sql script: 99_system.sql -# 15:05:54 > -# 15:05:54 > "/usr/bin/python2" "copy-into.Bug-3481.SQL.py" "copy-into.Bug-3481" -# 15:05:54 > +# 14:08:38 > +# 14:08:38 > "/usr/local/opt/python/bin/python2.7" "copy-into.Bug-3481.SQL.py" "copy-into.Bug-3481" +# 14:08:38 > -&4 f -&3 -&2 1 -1 -&4 t +a +b +c -&4 f -&3 -&2 1 -1 -&3 -&2 2 -1 -&4 t +# 14:08:39 > +# 14:08:39 > "Done." +# 14:08:39 > -&1 0 3 1 3 -% .tables # table_name -% name # name -% varchar # type -% 1 # length -[ "a" ] -[ "b" ] -[ "c" ] - -&3 - - -# 15:05:54 > -# 15:05:54 > "Done." -# 15:05:54 > - diff --git a/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out b/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out --- a/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out +++ b/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out @@ -61,25 +61,25 @@ Ready. # 15:17:35 > #select split_part('joeuser@mydatabase','@',1) AS "joeuser"; -% .L1 # table_name +% .L2 # table_name % joeuser # name % char # type % 18 # length [ "joeuser"] #select split_part('joeuser@mydatabase','@',2) AS "mydatabase"; -% .L1 # table_name +% .L2 # table_name % mydatabase # name % char # type % 18 # length [ "mydatabase" ] #select split
MonetDB: noconsole - console gone from implementation, mtest sti...
Changeset: 8ec2f8a7977b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ec2f8a7977b Modified Files: configure.ag monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_session.c sql/backends/monet5/mal_backend.c sql/backends/monet5/mal_backend.h sql/backends/monet5/sql_scenario.c tools/mserver/mserver5.c Branch: noconsole Log Message: console gone from implementation, mtest still needs a fix diffs (248 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -292,20 +292,6 @@ AC_ARG_ENABLE([pyintegration], [enable_pyintegration=$enableval], [enable_pyintegration=$dft_pyintegration]) -# The console is a direct client hooked onto the kernel with full -# administrative privileges, bypassing any security checks. It is -# handy only during development. -# We simply can't disable it for now because Testing gets hopelessly -# upset about it, but we can offer to the user to disable it for -# increased security. -dft_console=yes -AC_ARG_ENABLE([console], - [AS_HELP_STRING([--enable-console], - [enables direct console on the server (involves security risks) (default=yes)])], - [enable_console="$enableval"], - [enable_console=$dft_console]) -AS_VAR_IF([enable_console], [no], [], - [AC_DEFINE([HAVE_CONSOLE], 1, [If the console should be used])]) dft_int128=auto AC_ARG_ENABLE([int128], @@ -3049,7 +3035,6 @@ for comp in \ 'monetdb5' \ 'sql ' \ 'geom' \ - 'console ' \ 'pyintegration' \ 'rintegration' \ 'odbc' \ diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c --- a/monetdb5/mal/mal_client.c +++ b/monetdb5/mal/mal_client.c @@ -127,11 +127,7 @@ MCnewClient(void) { Client c; MT_lock_set(&mal_contextLock); - if (mal_clients[CONSOLE].user && mal_clients[CONSOLE].mode == FINISHCLIENT) { - /*system shutdown in progress */ - MT_lock_unset(&mal_contextLock); - return NULL; - } + // TODO: check some exiting flag here? for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++) { if (c->mode == FREECLIENT) { c->mode = RUNCLIENT; @@ -429,7 +425,7 @@ MCstopClients(Client cntxt) Client c = mal_clients; MT_lock_set(&mal_contextLock); - for(c= mal_clients +1; c < mal_clients+MAL_MAXCLIENTS; c++) + for(c= mal_clients ; c < mal_clients+MAL_MAXCLIENTS; c++) if( cntxt != c){ if ( c->mode == RUNCLIENT) c->mode = FINISHCLIENT; @@ -446,7 +442,7 @@ MCactiveClients(void) int freeclient=0, finishing=0, running=0, blocked = 0; Client cntxt = mal_clients; - for(cntxt= mal_clients+1; cntxtmode == FREECLIENT); finishing += (cntxt->mode == FINISHCLIENT); running += (cntxt->mode == RUNCLIENT); @@ -462,14 +458,8 @@ MCcloseClient(Client c) printf("closeClient %d " OIDFMT "\n", (int) (c - mal_clients), c->user); #endif /* free resources of a single thread */ - if (!isAdministrator(c)) { - freeClient(c); - return; - } + freeClient(c); - /* adm is set to disallow new clients entering */ - mal_clients[CONSOLE].mode = FINISHCLIENT; - mal_exit(); } str diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h --- a/monetdb5/mal/mal_client.h +++ b/monetdb5/mal/mal_client.h @@ -15,9 +15,6 @@ #include "mal_resolve.h" #include "mal_profiler.h" -#define CONSOLE 0 -#define isAdministrator(X) (X==mal_clients) - enum clientmode { FREECLIENT, FINISHCLIENT, diff --git a/monetdb5/mal/mal_session.c b/monetdb5/mal/mal_session.c --- a/monetdb5/mal/mal_session.c +++ b/monetdb5/mal/mal_session.c @@ -425,7 +425,7 @@ MSserveClient(void *dummy) Client c = (Client) dummy; str msg = 0; - if (!isAdministrator(c) && MCinitClientThread(c) < 0) { + if (MCinitClientThread(c) < 0) { MCcloseClient(c); return; } @@ -482,15 +482,8 @@ MSserveClient(void *dummy) assert(0); } - if (c->mode > FINISHCLIENT) { - if (isAdministrator(c) /* && moreClients(0)==0 */) { - if (c->scenario) { - exitScenario(c); - } - } - } - if (!isAdministrator(c)) - MCcloseClient(c); + MCcloseClient(c); + if (c->nspace && strcmp(c->nspace->name, "user") == 0) { GDKfree(c->nspace->space); GDKfree(c->nspace); diff --git a/sql/backends/monet5/mal_backend.c b/sql/backends/monet5/mal_backend.c --- a/sql/backends/monet5/mal_backend.c +++ b/sql/backends/monet5/mal_backend.c @@
MonetDB: noconsole - more server console special cases removed
Changeset: 006d512ba9b0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=006d512ba9b0 Modified Files: monetdb5/mal/mal.c monetdb5/mal/mal_authorize.c monetdb5/mal/mal_client.c monetdb5/mal/mal_debugger.c monetdb5/mal/mal_scenario.c monetdb5/mal/mal_session.c monetdb5/modules/mal/clients.c monetdb5/modules/mal/mal_mapi.c Branch: noconsole Log Message: more server console special cases removed diffs (264 lines): diff --git a/monetdb5/mal/mal.c b/monetdb5/mal/mal.c --- a/monetdb5/mal/mal.c +++ b/monetdb5/mal/mal.c @@ -144,14 +144,6 @@ void mserver_reset(int exit) */ mal_factory_reset(); mal_dataflow_reset(); - THRdel(mal_clients->mythread); - GDKfree(mal_clients->errbuf); - mal_clients->fdin->s = NULL; - bstream_destroy(mal_clients->fdin); - GDKfree(mal_clients->prompt); - GDKfree(mal_clients->username); - freeStack(mal_clients->glb); - freeSymbol(mal_clients->curprg); mal_client_reset(); mal_linker_reset(); mal_resource_reset(); diff --git a/monetdb5/mal/mal_authorize.c b/monetdb5/mal/mal_authorize.c --- a/monetdb5/mal/mal_authorize.c +++ b/monetdb5/mal/mal_authorize.c @@ -229,7 +229,7 @@ AUTHinitTables(const char *passwd) { char *pw; oid uid; Client c = &mal_clients[0]; - + // TODO: does this still work without the server console? if (passwd == NULL) passwd = "monetdb"; /* default password */ pw = mcrypt_BackendSum(passwd, strlen(passwd)); diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c --- a/monetdb5/mal/mal_client.c +++ b/monetdb5/mal/mal_client.c @@ -442,7 +442,7 @@ MCactiveClients(void) int freeclient=0, finishing=0, running=0, blocked = 0; Client cntxt = mal_clients; - for(cntxt= mal_clients; cntxtmode == FREECLIENT); finishing += (cntxt->mode == FINISHCLIENT); running += (cntxt->mode == RUNCLIENT); diff --git a/monetdb5/mal/mal_debugger.c b/monetdb5/mal/mal_debugger.c --- a/monetdb5/mal/mal_debugger.c +++ b/monetdb5/mal/mal_debugger.c @@ -388,18 +388,13 @@ mdbCommand(Client cntxt, MalBlkPtr mb, M char oldcmd[1024] = { 0 }; do { if (p != NULL) { - if (cntxt != mal_clients) - /* help mclients with fake prompt */ - if (lastcmd != 'l' && lastcmd != 'L') { - mnstr_printf(out, "mdb>"); - printTraceCall(out, mb, stk, pc, LIST_MAL_CALL); - } + /* help mclients with fake prompt */ + if (lastcmd != 'l' && lastcmd != 'L') { + mnstr_printf(out, "mdb>"); + printTraceCall(out, mb, stk, pc, LIST_MAL_CALL); + } + } - } - if (cntxt == mal_clients) { - cntxt->prompt = "mdb>"; - cntxt->promptlength = 4; - } if (cntxt->phase[MAL_SCENARIO_READER]) { retryRead: @@ -414,13 +409,7 @@ retryRead: goto retryRead; } } -#ifndef HAVE_EMBEDDED - else if (cntxt == mal_clients) { - /* switch to mdb streams */ - if (readConsole(cntxt) <= 0) - break; - } -#endif + b = CURRENT(cntxt); /* terminate the line with zero */ @@ -974,13 +963,13 @@ str mdbTrap(Client cntxt, MalBlkPtr mb, int cnt = 20; /* total 10 sec delay */ int pc = getPC(mb,p); - mnstr_printf(mal_clients[0].fdout, "#trapped %s.%s[%d]\n", + mnstr_printf(cntxt->fdout, "#trapped %s.%s[%d]\n", getModuleId(mb->stmt[0]), getFunctionId(mb->stmt[0]), pc); - printInstruction(mal_clients[0].fdout, mb, stk, p, LIST_MAL_DEBUG); + printInstruction(cntxt->fdout, mb, stk, p, LIST_MAL_DEBUG); cntxt->itrace = 'W'; MT_lock_set(&mal_contextLock); if (trapped_mb) { - mnstr_printf(mal_clients[0].fdout, "#registry not available\n"); + mnstr_printf(cntxt->fdout, "#registry not available\n"); mnstr_flush(cntxt->fdout); } while (trapped_mb && cnt-- > 0) { @@ -1015,12 +1004,12 @@ mdbStep(Client cntxt, MalBlkPtr mb, MalS state.p = getInstrPtr(mb, pc); state.pc = pc; cntxt->mdb = &state; - mnstr_printf(mal_clients[0].fdout, "#Process %d put to sleep\n", (int) (cntxt - mal_clients)); + mnstr_printf(out, "#Process %d put to sleep\n", (int) (cntxt
MonetDB: noconsole - startup messages don't have to start with #
Changeset: 99bcc306c7aa for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=99bcc306c7aa Modified Files: geom/lib/libgeom.c monetdb5/extras/rapi/rapi.c monetdb5/mal/mal_profiler.c monetdb5/modules/mal/clients.c monetdb5/modules/mal/mal_mapi.c sql/backends/monet5/UDF/pyapi/pyapi.c sql/backends/monet5/sql_scenario.c tools/mserver/mserver5.c Branch: noconsole Log Message: startup messages don't have to start with # diffs (139 lines): diff --git a/geom/lib/libgeom.c b/geom/lib/libgeom.c --- a/geom/lib/libgeom.c +++ b/geom/lib/libgeom.c @@ -23,7 +23,7 @@ libgeom_init(void) { initGEOS((GEOSMessageHandler) GDKerror, (GEOSMessageHandler) GDKerror); GEOS_setWKBByteOrder(1);/* NDR (little endian) */ - printf("# MonetDB/GIS module loaded\n"); + printf("MonetDB/GIS module loaded\n"); fflush(stdout); /* make merovingian see this *now* */ } diff --git a/monetdb5/extras/rapi/rapi.c b/monetdb5/extras/rapi/rapi.c --- a/monetdb5/extras/rapi/rapi.c +++ b/monetdb5/extras/rapi/rapi.c @@ -511,7 +511,7 @@ str RAPIprelude(void *ret) { } MT_lock_unset(&rapiLock); - printf("# MonetDB/R module loaded\n"); + printf("MonetDB/R module loaded\n"); } return MAL_SUCCEED; } diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c --- a/monetdb5/mal/mal_profiler.c +++ b/monetdb5/mal/mal_profiler.c @@ -1023,7 +1023,7 @@ void setHeartbeat(int delay) void initProfiler(void) { if( mal_trace) - openProfilerStream(mal_clients[0].fdout,0); + openProfilerStream(GDKout, 0); } void initHeartbeat(void) diff --git a/monetdb5/modules/mal/clients.c b/monetdb5/modules/mal/clients.c --- a/monetdb5/modules/mal/clients.c +++ b/monetdb5/modules/mal/clients.c @@ -613,7 +613,7 @@ CLTsessions(Client cntxt, MalBlkPtr mb, MT_lock_set(&mal_contextLock); -for (c = mal_clients + (GDKgetenv_isyes("monet_daemon") != 0); c < mal_clients + MAL_MAXCLIENTS; c++) +for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++) if (c->mode == RUNCLIENT) { BUNappend(user, c->username, FALSE); msg = MTIMEunix_epoch(&ts); diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c --- a/monetdb5/modules/mal/mal_mapi.c +++ b/monetdb5/modules/mal/mal_mapi.c @@ -433,7 +433,7 @@ static void SERVERannounce(struct in_add free(buf); else /* announce that we're now reachable */ - printf("# Listening for connection requests on " + printf("Listening for connection requests on " "mapi:monetdb://%s:%i/\n", host, port); } if (usockfile != NULL) { @@ -442,7 +442,7 @@ static void SERVERannounce(struct in_add free(buf); else /* announce that we're now reachable */ - printf("# Listening for UNIX domain connection requests on " + printf("Listening for UNIX domain connection requests on " "mapi:monetdb://%s\n", usockfile); } } diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c b/sql/backends/monet5/UDF/pyapi/pyapi.c --- a/sql/backends/monet5/UDF/pyapi/pyapi.c +++ b/sql/backends/monet5/UDF/pyapi/pyapi.c @@ -1288,7 +1288,7 @@ PyAPIprelude(void *ret) { pyapiInitialized++; } MT_lock_unset(&pyapiLock); -fprintf(stdout, "# MonetDB/Python module loaded\n"); +fprintf(stdout, "MonetDB/Python module loaded\n"); option_disable_fork = GDKgetenv_istrue(fork_disableflag) || GDKgetenv_isyes(fork_disableflag); return MAL_SUCCEED; } diff --git a/sql/backends/monet5/sql_scenario.c b/sql/backends/monet5/sql_scenario.c --- a/sql/backends/monet5/sql_scenario.c +++ b/sql/backends/monet5/sql_scenario.c @@ -169,7 +169,7 @@ SQLprelude(void *ret) if (tmp != MAL_SUCCEED) return (tmp); #ifndef HAVE_EMBEDDED - fprintf(stdout, "# MonetDB/SQL module loaded\n"); + fprintf(stdout, "MonetDB/SQL module loaded\n"); fflush(stdout); /* make merovingian see this *now* */ #endif /* only register availability of scenarios AFTER we are inited! */ diff --git a/tools/mserver/mserver5.c b/tools/mserver/mserver5.c --- a/tools/mserver/mserver5.c +++ b/tools/mserver/mserver5.c @@ -139,16 +139,16 @@ monet_hello(void) qi++; } - printf("# MonetDB 5 server v" VERSION); + printf("MonetDB 5 server v" VERSION); /* coverity[pointless_string_compare] */ if (strcmp(MONETDB_RELEASE, "unreleased") == 0) - printf("\n# This is an unreleased version"); + printf("\nThis is an unreleased version");
MonetDB: queryid2 - merge with default
Changeset: b14c4bcbf142 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b14c4bcbf142 Added Files: sql/test/BugTracker-2016/Tests/storagemodel.stable.out.Windows Removed Files: debian/libmonetdb5-server-gsl.install sql/backends/monet5/gsl/46_gsl.sql sql/backends/monet5/gsl/73_gsl.mal sql/backends/monet5/gsl/Makefile.ag sql/backends/monet5/gsl/Tests/All sql/backends/monet5/gsl/Tests/gsl00.malC sql/backends/monet5/gsl/Tests/gsl00.stable.err sql/backends/monet5/gsl/Tests/gsl00.stable.out sql/backends/monet5/gsl/Tests/gsl01.sql sql/backends/monet5/gsl/Tests/gsl01.stable.err sql/backends/monet5/gsl/Tests/gsl01.stable.out sql/backends/monet5/gsl/gsl.c sql/backends/monet5/gsl/gsl.mal sql/test/BugTracker-2016/Tests/storagemodel.stable.out.32bit sql/test/BugTracker-2016/Tests/storagemodel.stable.out.int128 Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc clients/Tests/All clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc configure.ag debian/changelog debian/control debian/rules gdk/libbat.rc libversions monetdb5/mal/mal_resource.h monetdb5/tools/libmonetdb5.rc sql/backends/monet5/Makefile.ag sql/backends/monet5/sql.c sql/test/BugTracker-2013/Tests/All sql/test/BugTracker-2015/Tests/All sql/test/BugTracker-2016/Tests/storagemodel.sql sql/test/BugTracker-2016/Tests/storagemodel.stable.err sql/test/BugTracker-2016/Tests/storagemodel.stable.out sql/test/Tests/All sql/test/Tests/systemfunctions.stable.out sql/test/Tests/systemfunctions.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/All sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/All sql/test/emptydb-upgrade-chain/Tests/check.stable.out sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-hge/Tests/All sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade/Tests/All sql/test/emptydb-upgrade/Tests/check.stable.out sql/test/emptydb-upgrade/Tests/check.stable.out.32bit sql/test/emptydb-upgrade/Tests/check.stable.out.int128 sql/test/emptydb/Tests/All sql/test/emptydb/Tests/check.stable.out sql/test/emptydb/Tests/check.stable.out.32bit sql/test/emptydb/Tests/check.stable.out.int128 sql/test/leaks/Tests/All sql/test/leaks/Tests/check1.stable.out sql/test/leaks/Tests/check1.stable.out.int128 sql/test/leaks/Tests/check2.stable.out sql/test/leaks/Tests/check2.stable.out.int128 sql/test/leaks/Tests/check3.stable.out sql/test/leaks/Tests/check3.stable.out.int128 sql/test/leaks/Tests/check4.stable.out sql/test/leaks/Tests/check4.stable.out.int128 sql/test/leaks/Tests/check5.stable.out sql/test/leaks/Tests/check5.stable.out.int128 sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.sql sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.stable.out sql/test/sys-schema/Tests/check_PrimaryKey_uniqueness.sql sql/test/sys-schema/Tests/check_PrimaryKey_uniqueness.stable.out testing/Mtest.py.in vertoo.data Branch: queryid2 Log Message: merge with default diffs (truncated from 3384 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -649,3 +649,5 @@ ea0fe76e0095771bccca626b39e3e839a90ba320 afc859894553ef804cded49a83b035d9d2cda73b Dec2016_release bf4b58d202645774b75c7f508c84b30e3be78a49 Dec2016_5 bf4b58d202645774b75c7f508c84b30e3be78a49 Dec2016_SP1_release +3968ed7fb6dab04a2530c6c3e1a5ffd6d649298b Dec2016_7 +3968ed7fb6dab04a2530c6c3e1a5ffd6d649298b Dec2016_SP2_release diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -78,6 +78,14 @@ %endif %endif +# If the _without_pcre macro is not set, the PCRE library is used for +# the implementation of the SQL LIKE and ILIKE operators. Otherwise +# the POS
MonetDB: default - merged in queryid2
Changeset: 1a3bd9a93d80 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1a3bd9a93d80 Modified Files: monetdb5/mal/mal_runtime.c sql/backends/monet5/sql.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_result.h sql/include/sql_catalog.h sql/storage/bat/res_table.c sql/storage/sql_storage.h sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128 sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out.int128 sql/test/Tests/identifiers.stable.out sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out sql/test/VOC/Tests/median.Bug-3096.stable.out sql/test/bugs/Tests/aggr_single_value.stable.out sql/test/bugs/Tests/aggr_single_value.stable.out.int128 sql/test/pg_regress/Tests/date.stable.out sql/test/pg_regress/Tests/interval.stable.out sql/test/pg_regress/Tests/interval.stable.out.int128 sql/test/pg_regress/Tests/strings.stable.out sql/test/pg_regress/Tests/strings_concat.stable.out Branch: default Log Message: merged in queryid2 diffs (truncated from 1014 to 300 lines): diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c --- a/monetdb5/mal/mal_runtime.c +++ b/monetdb5/mal/mal_runtime.c @@ -83,6 +83,7 @@ runtimeProfileInit(Client cntxt, MalBlkP if (i == qtop) { QRYqueue[i].mb = mb; QRYqueue[i].tag = qtag++; + mb->tag = QRYqueue[i].tag; QRYqueue[i].stk = stk; // for status pause 'p'/running '0'/ quiting 'q' QRYqueue[i].start = (lng)time(0); QRYqueue[i].runtime = mb->runtime; // the estimated execution time diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -1705,7 +1705,7 @@ mvc_result_set_wrap( Client cntxt, MalBl b = BATdescriptor(bid); if ( b == NULL) throw(MAL,"sql.resultset","Failed to access order column"); - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, b); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 5), 1, b); if (res < 0) msg = createException(SQL, "sql.resultSet", "failed"); BBPunfix(b->batCacheid); @@ -1796,7 +1796,7 @@ mvc_export_table_wrap( Client cntxt, Mal order = BATdescriptor(bid); if ( order == NULL) throw(MAL,"sql.resultset","Failed to access order column"); - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, order); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 11), 1, order); t = m->results; if (res < 0){ msg = createException(SQL, "sql.resultSet", "failed"); @@ -1895,7 +1895,7 @@ mvc_row_result_wrap( Client cntxt, MalBl return msg; if ((msg = checkSQLContext(cntxt)) != NULL) return msg; - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, NULL); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 5), 1, NULL); tbl = BATdescriptor(tblId); atr = BATdescriptor(atrId); @@ -1970,7 +1970,7 @@ mvc_export_row_wrap( Client cntxt, MalBl return msg; if ((msg = checkSQLContext(cntxt)) != NULL) return msg; - res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, NULL); + res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 11), 1, NULL); t = m->results; if (res < 0){ @@ -2071,7 +2071,7 @@ mvc_table_result_wrap(Client cntxt, MalB if ((order = BATdescriptor(*order_bid)) == NULL) { throw(SQL, "sql.resultSet", "Cannot access descriptor"); } - *res_id = mvc_result_table(m, *nr_cols, *qtype, order); + *res_id = mvc_result_table(m, mb->tag, *nr_cols, *qtype, order); if (*res_id < 0) res = createException(SQL, "sql.resultSet", "failed"); BBPunfix(or
MonetDB: queryid2 - closing branch
Changeset: 53e82e18e17b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=53e82e18e17b Branch: queryid2 Log Message: closing branch ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - finally fixing R test cases
Changeset: 65191bb72656 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=65191bb72656 Modified Files: clients/R/Tests/copy_into_fwf.stable.out clients/R/Tests/dbapply.stable.out clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.err clients/R/Tests/dbi.stable.out clients/R/Tests/dplyr-flights.R clients/R/Tests/dplyr-flights.stable.out clients/R/Tests/dplyr.R clients/R/Tests/dplyr.stable.err clients/R/Tests/dplyr.stable.out Branch: Dec2016 Log Message: finally fixing R test cases diffs (truncated from 361 to 300 lines): diff --git a/clients/R/Tests/copy_into_fwf.stable.out b/clients/R/Tests/copy_into_fwf.stable.out --- a/clients/R/Tests/copy_into_fwf.stable.out +++ b/clients/R/Tests/copy_into_fwf.stable.out @@ -24,11 +24,8 @@ Ready. # 16:40:23 > "R" "--vanilla" "--slave" "--args" "30658" # 16:40:23 > -[1] TRUE SQL CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION) - ROWS Fetched: 0 [complete] - Changed: NA mpg cyl disp hp dratwt qsec vs am gear carb 1 21.0 6 160.0 110 3.90 2.620 16.46 0 144 2 21.0 6 160.0 110 3.90 2.875 17.02 0 144 @@ -62,7 +59,6 @@ 29 15.8 8 351.0 264 4.22 3.170 14.50 30 19.7 6 145.0 175 3.62 2.770 15.50 0 156 31 15.0 8 301.0 335 3.54 3.570 14.60 0 158 32 21.4 4 121.0 109 4.11 2.780 18.60 1 142 -[1] TRUE [1] "SUCCESS" # 16:40:23 > diff --git a/clients/R/Tests/dbapply.stable.out b/clients/R/Tests/dbapply.stable.out --- a/clients/R/Tests/dbapply.stable.out +++ b/clients/R/Tests/dbapply.stable.out @@ -29,14 +29,10 @@ Ready. # 11:43:14 > "R" "--vanilla" "--slave" "--args" "35780" # 11:43:14 > -[1] TRUE -[1] TRUE -[1] TRUE [1] 32 [1] TRUE [1] 32 [1] 42 -[1] TRUE [1] "SUCCESS" # 11:43:21 > diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R --- a/clients/R/Tests/dbi.R +++ b/clients/R/Tests/dbi.R @@ -43,6 +43,7 @@ dbRemoveTable(con,tname) stopifnot(identical(dbExistsTable(con,tname),FALSE)) # write test table iris +# iris is one of the built-in datasets in R data(iris) dbWriteTable(con,tname,iris) @@ -65,16 +66,22 @@ stopifnot(identical(res@env$success,TRUE stopifnot(dbColumnInfo(res)[[1,1]] == "Species") stopifnot(dbColumnInfo(res)[[2,1]] == "Sepal.Width") -stopifnot(dbGetRowCount(res) == 150 && res@env$info$rows == 150) +stopifnot(dbGetRowCount(res) == 0) data <- dbFetch(res,10) +stopifnot(dbGetRowCount(res) == 10) + + stopifnot(dim(data)[[1]] == 10) stopifnot(dim(data)[[2]] == 2) stopifnot(res@env$delivered == 10) stopifnot(dbHasCompleted(res) == FALSE) data2 <- dbFetch(res,-1) + +stopifnot(dbGetRowCount(res) == 150) + stopifnot(dim(data2)[[1]] == 140) stopifnot(dbHasCompleted(res) == TRUE) diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err --- a/clients/R/Tests/dbi.stable.err +++ b/clients/R/Tests/dbi.stable.err @@ -37,6 +37,7 @@ Use 'dbIsValid' instead. See help("Deprecated") Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. +Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Treating character vector parameter as file name(s) for monetdb.read.csv() Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. diff --git a/clients/R/Tests/dbi.stable.out b/clients/R/Tests/dbi.stable.out --- a/clients/R/Tests/dbi.stable.out +++ b/clients/R/Tests/dbi.stable.out @@ -44,42 +44,12 @@ Ready. # loading sql script: 90_generator.sql # loading sql script: 99_system.sql -# 12:26:31 > -# 12:26:31 > "R" "--vanilla" "--slave" "--args" "31372" -# 12:26:31 > +# 15:00:13 > +# 15:00:13 > "R" "--vanilla" "--slave" "--args" "34632" +# 15:00:13 > -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE [1] "\"Need to quote this table name\"" "monetdbtest" [3] "othermethod" -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE -[1] TRUE [1] "SUCCESS" # 12:26:33 > diff --git
MonetDB: Dec2016 - added survey dependency
Changeset: 528fecb20eee for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=528fecb20eee Modified Files: clients/R/Tests/deps-test.R Branch: Dec2016 Log Message: added survey dependency diffs (10 lines): diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R --- a/clients/R/Tests/deps-test.R +++ b/clients/R/Tests/deps-test.R @@ -4,5 +4,5 @@ dd <- capture.output(suppressMessages(su repos <- 'http://cran.rstudio.com/' if(length(np)) install.packages(np, repos=repos, quiet=T) update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T) - })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata')) + })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata', 'survey')) }))) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - forcing dplyr dev
Changeset: 8e3dccb2c60f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8e3dccb2c60f Modified Files: clients/R/Tests/deps-test.R Branch: Dec2016 Log Message: forcing dplyr dev diffs (16 lines): diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R --- a/clients/R/Tests/deps-test.R +++ b/clients/R/Tests/deps-test.R @@ -5,4 +5,12 @@ dd <- capture.output(suppressMessages(su if(length(np)) install.packages(np, repos=repos, quiet=T) update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T) })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata', 'survey')) + + # dev dplyr + if (packageVersion("devtools") < 1.6) { + install.packages("devtools") + } + devtools::install_github("hadley/lazyeval") + devtools::install_github("hadley/dplyr") + }))) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - unchecked malloc, found with mallocs.SQL.py
Changeset: ebff12da64d1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ebff12da64d1 Modified Files: monetdb5/mal/mal_instruction.c monetdb5/optimizer/opt_pipes.c monetdb5/optimizer/opt_support.c sql/backends/monet5/sql_execute.c Branch: default Log Message: unchecked malloc, found with mallocs.SQL.py diffs (74 lines): diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -392,10 +392,9 @@ newInstruction(MalBlkPtr mb, str modnme, InstrPtr copyInstruction(InstrPtr p) { - InstrPtr new; - new = (InstrPtr) GDKmalloc(offsetof(InstrRecord, argv) + p->maxarg * sizeof(p->maxarg)); - if( new == NULL) { - GDKerror("copyInstruction:failed to allocated space"); + InstrPtr new = (InstrPtr) GDKmalloc(offsetof(InstrRecord, argv) + p->maxarg * sizeof(p->maxarg)); + if(new == NULL) { + GDKerror("copyInstruction: failed to allocated space"); return new; } oldmoveInstruction(new, p); diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c --- a/monetdb5/optimizer/opt_pipes.c +++ b/monetdb5/optimizer/opt_pipes.c @@ -495,6 +495,9 @@ addOptimizerPipe(Client cntxt, MalBlkPtr if (pipes[i].mb) { for (j = 1; j < pipes[i].mb->stop - 1; j++) { p = copyInstruction(pipes[i].mb->stmt[j]); + if (!p) { // oh malloc you cruel mistress + throw(MAL, "optimizer.addOptimizerPipe", "Out of memory"); + } for (k = 0; k < p->argc; k++) getArg(p, k) = cloneVariable(mb, pipes[i].mb, getArg(p, k)); typeChecker(cntxt->fdout, cntxt->nspace, mb, p, FALSE); diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c --- a/monetdb5/optimizer/opt_support.c +++ b/monetdb5/optimizer/opt_support.c @@ -124,8 +124,8 @@ optimizeMALBlock(Client cntxt, MalBlkPtr /* force at least once a complete type check by resetting the type check flag */ - resetMalBlk(mb,mb->stop); - chkProgram(cntxt->fdout,cntxt->nspace,mb); + resetMalBlk(mb, mb->stop); + chkProgram(cntxt->fdout, cntxt->nspace,mb); if (mb->errors) throw(MAL, "optimizer.MALoptimizer", "Start with inconsistent MAL plan"); @@ -133,7 +133,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr * When no optimzer call is found, be terminate. */ do { qot = 0; - for (pc = 0; pc < mb->stop ; pc++) { + for (pc = 0; pc < mb->stop; pc++) { p = getInstrPtr(mb, pc); if (getModuleId(p) == optimizerRef && p->fcn && p->token != REMsymbol) { /* all optimizers should behave like patterns */ @@ -143,7 +143,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr msg = (str) (*p->fcn) (cntxt, mb, 0, p); if (msg) { str place = getExceptionPlace(msg); - str nmsg= createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg)); + str nmsg = createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg)); GDKfree(place); GDKfree(msg); msg = nmsg; diff --git a/sql/backends/monet5/sql_execute.c b/sql/backends/monet5/sql_execute.c --- a/sql/backends/monet5/sql_execute.c +++ b/sql/backends/monet5/sql_execute.c @@ -310,7 +310,7 @@ SQLrun(Client c, backend *be, mvc *m){ // This include template constants, BAT sizes. if( m->emod & mod_debug) mb->keephistory = TRUE; - msg = SQLoptimizeQuery(c,mb); + msg = SQLoptimizeQuery(c, mb); mb->keephistory = FALSE; if( mb->errors){ ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - more unchecked malloc, this time in optimizer...
Changeset: e18c02b1ad0e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e18c02b1ad0e Modified Files: monetdb5/optimizer/opt_dataflow.c monetdb5/optimizer/opt_support.c monetdb5/optimizer/opt_wrapper.c sql/backends/monet5/sql_execute.c Branch: default Log Message: more unchecked malloc, this time in optimizers. Changed contract for optimizer function return code, values < 0 signal error and abort query. diffs (86 lines): diff --git a/monetdb5/optimizer/opt_dataflow.c b/monetdb5/optimizer/opt_dataflow.c --- a/monetdb5/optimizer/opt_dataflow.c +++ b/monetdb5/optimizer/opt_dataflow.c @@ -201,9 +201,10 @@ OPTdataflowImplementation(Client cntxt, limit= mb->stop; slimit= mb->ssize; old = mb->stmt; - if ( newMalBlkStmt(mb, mb->ssize) <0 ) + if (newMalBlkStmt(mb, mb->ssize) < 0) { + actions = -1; goto wrapup; - + } pushInstruction(mb,old[0]); /* inject new dataflow barriers using a single pass through the program */ @@ -320,9 +321,9 @@ OPTdataflowImplementation(Client cntxt, newComment(mb,buf); wrapup: - if( states) GDKfree(states); - if( sink) GDKfree(sink); - if( old) GDKfree(old); + if(states) GDKfree(states); + if(sink) GDKfree(sink); + if(old)GDKfree(old); return actions; } diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c --- a/monetdb5/optimizer/opt_support.c +++ b/monetdb5/optimizer/opt_support.c @@ -125,7 +125,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr /* force at least once a complete type check by resetting the type check flag */ resetMalBlk(mb, mb->stop); - chkProgram(cntxt->fdout, cntxt->nspace,mb); + chkProgram(cntxt->fdout, cntxt->nspace, mb); if (mb->errors) throw(MAL, "optimizer.MALoptimizer", "Start with inconsistent MAL plan"); diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c --- a/monetdb5/optimizer/opt_wrapper.c +++ b/monetdb5/optimizer/opt_wrapper.c @@ -143,12 +143,15 @@ str OPTwrapper (Client cntxt, MalBlkPtr } else if( p ) removeInstruction(mb, p); - for ( i=0; codes[i].nme; i++) - if ( strcmp(codes[i].nme, optimizer)== 0 ){ - actions = (int)(*(codes[i].fcn))(cntxt, mb, stk,0); + for (i=0; codes[i].nme; i++) + if (strcmp(codes[i].nme, optimizer) == 0){ + actions = (int)(*(codes[i].fcn))(cntxt, mb, stk, 0); + if (actions < 1) { + throw(MAL, optimizer, "Error in optimizer %s", optimizer); + } break; } - if ( codes[i].nme == 0) + if (codes[i].nme == 0) throw(MAL, optimizer, "Optimizer implementation '%s' missing", fcnnme); OPTIMIZERDEBUG { diff --git a/sql/backends/monet5/sql_execute.c b/sql/backends/monet5/sql_execute.c --- a/sql/backends/monet5/sql_execute.c +++ b/sql/backends/monet5/sql_execute.c @@ -276,7 +276,7 @@ SQLrun(Client c, backend *be, mvc *m){ /* only consider a re-optimization when we are dealing with query templates */ for ( i= 1; i < mb->stop;i++){ - p=getInstrPtr(mb,i); + p = getInstrPtr(mb,i); if( getFunctionId(p) && qc_isapreparedquerytemplate(getFunctionId(p) ) ){ msg = SQLexecutePrepared(c, be, p->blk); freeMalBlk(mb); @@ -284,6 +284,9 @@ SQLrun(Client c, backend *be, mvc *m){ } if( getFunctionId(p) && p->blk && qc_isaquerytemplate(getFunctionId(p)) ) { mc = copyMalBlk(p->blk); + if (!mc) { + throw(SQL, "sql.prepare", "Out of memory"); + } retc = p->retc; freeMalBlk(mb); mb = mc; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - more malloc fail checks and re-enabled test case
Changeset: 7c0e4baa6fed for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7c0e4baa6fed Modified Files: monetdb5/mal/mal_instruction.c monetdb5/optimizer/opt_wrapper.c sql/backends/monet5/sql_execute.c sql/test/malloc_fail/Tests/All sql/test/malloc_fail/Tests/mallocs.SQL.py Branch: default Log Message: more malloc fail checks and re-enabled test case diffs (110 lines): diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -263,13 +263,17 @@ copyMalBlk(MalBlkPtr old) // copy all variable records for (i = 0; i < old->vtop; i++) { mb->var[i]= old->var[i]; - VALcopy(&(mb->var[i].value), &(old->var[i].value)); + if (!VALcopy(&(mb->var[i].value), &(old->var[i].value))) { + GDKfree(mb); + GDKerror("copyMalBlk:" MAL_MALLOC_FAIL); + return NULL; + } } mb->stmt = (InstrPtr *) GDKzalloc(sizeof(InstrPtr) * old->ssize); if (mb->stmt == NULL) { - GDKfree(mb->var); + GDKfree(mb->var); // this leaks strings in var GDKfree(mb); GDKerror("copyMalBlk:" MAL_MALLOC_FAIL); return NULL; @@ -278,10 +282,20 @@ copyMalBlk(MalBlkPtr old) mb->stop = old->stop; mb->ssize = old->ssize; assert(old->stop < old->ssize); - for (i = 0; i < old->stop; i++) + for (i = 0; i < old->stop; i++) { mb->stmt[i] = copyInstruction(old->stmt[i]); - + if(!mb->stmt[i]) { + GDKfree(mb); + GDKerror("copyMalBlk:" MAL_MALLOC_FAIL); + return NULL; + } + } mb->help = old->help ? GDKstrdup(old->help) : NULL; + if (old->help && !mb->help) { + GDKfree(mb); + GDKerror("copyMalBlk:" MAL_MALLOC_FAIL); + return NULL; + } strncpy(mb->binding, old->binding, IDLENGTH); mb->errors = old->errors; mb->tag = old->tag; diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c --- a/monetdb5/optimizer/opt_wrapper.c +++ b/monetdb5/optimizer/opt_wrapper.c @@ -146,7 +146,7 @@ str OPTwrapper (Client cntxt, MalBlkPtr for (i=0; codes[i].nme; i++) if (strcmp(codes[i].nme, optimizer) == 0){ actions = (int)(*(codes[i].fcn))(cntxt, mb, stk, 0); - if (actions < 1) { + if (actions < 0) { throw(MAL, optimizer, "Error in optimizer %s", optimizer); } break; diff --git a/sql/backends/monet5/sql_execute.c b/sql/backends/monet5/sql_execute.c --- a/sql/backends/monet5/sql_execute.c +++ b/sql/backends/monet5/sql_execute.c @@ -271,6 +271,9 @@ SQLrun(Client c, backend *be, mvc *m){ return createException(PARSE, "SQLparser", "%s", m->errstr); // locate and inline the query template instruction mb = copyMalBlk(c->curprg->def); + if (!mb) { + throw(SQL, "sql.prepare", "Out of memory"); + } mb->history = c->curprg->def->history; c->curprg->def->history = 0; diff --git a/sql/test/malloc_fail/Tests/All b/sql/test/malloc_fail/Tests/All --- a/sql/test/malloc_fail/Tests/All +++ b/sql/test/malloc_fail/Tests/All @@ -1,1 +1,1 @@ -# mallocs not run for now since it hangs mserver +# mallocs still hangs diff --git a/sql/test/malloc_fail/Tests/mallocs.SQL.py b/sql/test/malloc_fail/Tests/mallocs.SQL.py --- a/sql/test/malloc_fail/Tests/mallocs.SQL.py +++ b/sql/test/malloc_fail/Tests/mallocs.SQL.py @@ -1,6 +1,10 @@ import pymonetdb import os +import logging +logging.basicConfig() + + dbh = pymonetdb.connect(database = os.environ['TSTDB'], port = int(os.environ['MAPIPORT']), hostname = "localhost") @@ -15,14 +19,15 @@ create procedure setmallocsuccesscount(c call setmallocsuccesscount(%d); SELECT * FROM tables; """ -i = 3300 +i = 3000 + while i > 1000: -#print(i) +print(i) i-=1 try: cursor.execute(q % (i)) except Exception as e: -#print(e) + # print(e) pass finally: dbh.rollback() ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - more unchecked malloc failures
Changeset: 2fcd5e98b30e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2fcd5e98b30e Modified Files: monetdb5/mal/mal_builder.c monetdb5/optimizer/opt_deadcode.c monetdb5/optimizer/opt_evaluate.c monetdb5/optimizer/opt_support.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_gencode.c Branch: default Log Message: more unchecked malloc failures diffs (108 lines): diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c --- a/monetdb5/mal/mal_builder.c +++ b/monetdb5/mal/mal_builder.c @@ -85,7 +85,7 @@ newComment(MalBlkPtr mb, const char *val InstrPtr q = newInstruction(mb, NULL, NULL); ValRecord cst; - if ( q == NULL) + if (q == NULL) return NULL; q->token = REMsymbol; q->barrier = 0; diff --git a/monetdb5/optimizer/opt_deadcode.c b/monetdb5/optimizer/opt_deadcode.c --- a/monetdb5/optimizer/opt_deadcode.c +++ b/monetdb5/optimizer/opt_deadcode.c @@ -34,8 +34,10 @@ OPTdeadcodeImplementation(Client cntxt, limit = mb->stop; slimit = mb->ssize; - if (newMalBlkStmt(mb, mb->ssize) < 0) + if (newMalBlkStmt(mb, mb->ssize) < 0) { + actions = -1; goto wrapup; + } // Calculate the instructions in which a variable is used. // Variables can be used multiple times in an instruction. diff --git a/monetdb5/optimizer/opt_evaluate.c b/monetdb5/optimizer/opt_evaluate.c --- a/monetdb5/optimizer/opt_evaluate.c +++ b/monetdb5/optimizer/opt_evaluate.c @@ -179,7 +179,11 @@ OPTevaluateImplementation(Client cntxt, profiler = malProfileMode; /* we don't trace it */ malProfileMode = 0; if ( env == NULL) { - env = prepareMALstack(mb, 2 * mb->vsize ); + env = prepareMALstack(mb, 2 * mb->vsize); + if (!env) { + actions = -1; + goto wrapup; + } env->keepAlive = TRUE; } msg = reenterMAL(cntxt, mb, i, i + 1, env); diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c --- a/monetdb5/optimizer/opt_support.c +++ b/monetdb5/optimizer/opt_support.c @@ -144,9 +144,11 @@ optimizeMALBlock(Client cntxt, MalBlkPtr if (msg) { str place = getExceptionPlace(msg); str nmsg = createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg)); - GDKfree(place); - GDKfree(msg); - msg = nmsg; + if (nmsg && place) { + GDKfree(msg); + msg = nmsg; + GDKfree(place); + } goto wrapup; } if (cntxt->mode == FINISHCLIENT) @@ -158,11 +160,14 @@ optimizeMALBlock(Client cntxt, MalBlkPtr wrapup: /* Keep the total time spent on optimizing the plan for inspection */ - if( actions){ - mb->optimize= GDKusec() - clk; - snprintf(buf, 256, "%-20s actions=%2d time=" LLFMT " usec", "total",actions, mb->optimize); + if(actions > 0 && msg == MAL_SUCCEED){ + mb->optimize = GDKusec() - clk; + snprintf(buf, 256, "%-20s actions=%2d time=" LLFMT " usec", "total", actions, mb->optimize); newComment(mb, buf); } + if (msg != MAL_SUCCEED) { + mb->errors++; + } if (cnt >= mb->stop) throw(MAL, "optimizer.MALoptimizer", OPTIMIZER_CYCLE); return msg; diff --git a/sql/backends/monet5/sql_execute.c b/sql/backends/monet5/sql_execute.c --- a/sql/backends/monet5/sql_execute.c +++ b/sql/backends/monet5/sql_execute.c @@ -319,8 +319,9 @@ SQLrun(Client c, backend *be, mvc *m){ msg = SQLoptimizeQuery(c, mb); mb->keephistory = FALSE; - if( mb->errors){ - freeMalBlk(mb); + if (mb->errors){ + //freeMalBlk(mb); + // mal block might be so broken free causes segfault return msg; } diff --git a/sql/backends/monet5/sql_gencode.c b/sql/backends/monet5/sql_gencode.c --- a/sql/backends/monet5/sql_gencode.c +++ b/sql/backends/monet5/sql_gencode.c @@ -662,6 +662,10 @@ backend_call(backend *be, Client c, cq * MalBlkPtr mb = c->curprg->def; q = newStmt(mb, userRef, cq->name); +
MonetDB: default - Added markers to invocations of GDKmalloc and...
Changeset: 34747c9d31c1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=34747c9d31c1 Modified Files: gdk/gdk_logger.c gdk/gdk_value.c geom/monetdb5/geom.c monetdb5/mal/mal_builder.c monetdb5/modules/atoms/blob.c monetdb5/modules/atoms/inet.c monetdb5/modules/atoms/uuid.c monetdb5/modules/atoms/xml.c monetdb5/modules/mal/calc.c monetdb5/modules/mal/tablet.c monetdb5/modules/mal/txtsim.c monetdb5/optimizer/opt_mergetable.c sql/backends/monet5/UDF/pyapi/emit.c sql/backends/monet5/mal_backend.c sql/backends/monet5/rel_bin.c sql/backends/monet5/sql.c sql/backends/monet5/sql_execute.c sql/backends/monet5/sql_gencode.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statement.c sql/backends/monet5/sql_statistics.c sql/backends/monet5/sql_upgrades.c sql/backends/monet5/vaults/fits/fits.c sql/backends/monet5/vaults/netcdf/netcdf.c sql/common/sql_keyword.c sql/common/sql_mem.c sql/common/sql_string.c sql/server/rel_updates.c sql/server/sql_atom.c sql/server/sql_mvc.c sql/server/sql_qc.c sql/server/sql_scan.c sql/storage/bat/bat_table.c sql/storage/bat/res_table.c sql/storage/store.c sql/storage/store_sequence.c tools/mserver/mserver5.c Branch: default Log Message: Added markers to invocations of GDKmalloc and friends where return value is not checked. (//FIXME unchecked_malloc ...) diffs (truncated from 803 to 300 lines): diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c --- a/gdk/gdk_logger.c +++ b/gdk/gdk_logger.c @@ -345,7 +345,10 @@ log_read_updates(logger *lg, trans *tr, if (tt < TYPE_str) tv = lg->buf; else if (tt > TYPE_str) + // FIXME unchecked_malloc ATOMnil can return NULL + tv = ATOMnil(tt); + assert(l->nr <= (lng) BUN_MAX); if (l->flag == LOG_UPDATE) { uid = COLnew(0, ht, (BUN) l->nr, PERSISTENT); @@ -372,6 +375,8 @@ log_read_updates(logger *lg, trans *tr, } } else { void *(*rh) (ptr, stream *, size_t) = ht == TYPE_void ? BATatoms[TYPE_oid].atomRead : BATatoms[ht].atomRead; + // FIXME unchecked_malloc ATOMnil can return NULL + void *hv = ATOMnil(ht); for (; l->nr > 0; l->nr--) { diff --git a/gdk/gdk_value.c b/gdk/gdk_value.c --- a/gdk/gdk_value.c +++ b/gdk/gdk_value.c @@ -143,6 +143,7 @@ VALcopy(ValPtr d, const ValRecord *s) if (!ATOMextern(s->vtype)) { *d = *s; } else if (s->val.pval == 0) { + // FIXME unchecked_malloc ATOMnil can return NULL d->val.pval = ATOMnil(s->vtype); d->vtype = s->vtype; } else if (s->vtype == TYPE_str) { diff --git a/geom/monetdb5/geom.c b/geom/monetdb5/geom.c --- a/geom/monetdb5/geom.c +++ b/geom/monetdb5/geom.c @@ -1745,7 +1745,7 @@ dumpPointsPoint(BAT *idBAT, BAT *geomBAT str err = MAL_SUCCEED; (*lvl)++; - + // FIXME unchecked_malloc GDKmalloc can return NULL newPath = GDKmalloc(pathLength + lvlDigitsNum + 1); sprintf(newPath, "%s%u", path, *lvl); @@ -1805,7 +1805,7 @@ dumpPointsPolygon(BAT *idBAT, BAT *geomB throw(MAL, "geom.DumpPoints", "GEOSGetExteriorRing failed"); (*lvl)++; - + // FIXME unchecked_malloc GDKmalloc can return NULL newPath = GDKmalloc(pathLength + lvlDigitsNum + extraLength + 1); sprintf(newPath, "%s%u%s", path, *lvl, extraStr); @@ -1858,6 +1858,7 @@ dumpPointsMultiGeometry(BAT *idBAT, BAT multiGeometry = GEOSGetGeometryN(geosGeometry, i); lvl++; + // FIXME unchecked_malloc GDKmalloc can return NULL newPath = GDKmalloc(pathLength + lvlDigitsNum + extraLength + 1); sprintf(newPath, "%s%u%s", path, lvl, extraStr); @@ -2165,6 +2166,7 @@ wkbFROMSTR_withSRID(char *geomWKT, int * if (strncasecmp(geomWKT, polyhedralSurface, strlen(polyhedralSurface)) == 0) { size_t sizeOfInfo = strlen(geomWKT) - strlen(polyhedralSurface); geomWKT_original = geomWKT; + // FIXME unchecked_malloc GDKmalloc can return NULL geomWKT = GDKmalloc(sizeOfInfo + strlen(multiPolygon) + 1); strcpy(geomWKT, multiPolygon); memcpy(geomWKT + strlen(multiPolygon), &geomWKT_original[strlen(polyhedralSurface)], sizeOfInfo); @@ -2219,7 +2221,7 @@ wkbaFROMSTR_withSRID(char *fromStr, int //read the number of items from the beginning of the string memcpy(&items, fromStr, sizeof(int)); skipBytes += size
MonetDB: default - fix compilation
Changeset: 43013a1ede09 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=43013a1ede09 Modified Files: monetdb5/modules/mal/txtsim.c Branch: default Log Message: fix compilation diffs (12 lines): diff --git a/monetdb5/modules/mal/txtsim.c b/monetdb5/modules/mal/txtsim.c --- a/monetdb5/modules/mal/txtsim.c +++ b/monetdb5/modules/mal/txtsim.c @@ -27,7 +27,7 @@ if (b) {\ if (ATOMextern(t)) {\ *(ptr*) res = (ptr) ATOMnil(t);\ - // FIXME unchecked_malloc ATOMnil can return NULL \ + /* FIXME unchecked_malloc ATOMnil can return NULL */ \ } else {\ memcpy(res, ATOMnilptr(t), ATOMsize(t));\ }\ ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - more unchecked mallocs
Changeset: 5d806584f589 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d806584f589 Modified Files: sql/backends/monet5/sql_statement.c sql/storage/store_dependency.c Branch: default Log Message: more unchecked mallocs diffs (38 lines): diff --git a/sql/backends/monet5/sql_statement.c b/sql/backends/monet5/sql_statement.c --- a/sql/backends/monet5/sql_statement.c +++ b/sql/backends/monet5/sql_statement.c @@ -2306,6 +2306,7 @@ stmt_output(backend *be, stmt *lst) const char *ntn = sql_escape_ident(tn); const char *nsn = sql_escape_ident(sn); size_t fqtnl = strlen(ntn) + 1 + strlen(nsn) + 1; + // FIXME unchecked_malloc NEW_ARRAY can return NULL char *fqtn = NEW_ARRAY(char, fqtnl); snprintf(fqtn, fqtnl, "%s.%s", nsn, ntn); diff --git a/sql/storage/store_dependency.c b/sql/storage/store_dependency.c --- a/sql/storage/store_dependency.c +++ b/sql/storage/store_dependency.c @@ -111,6 +111,7 @@ sql_trans_get_dependencies(sql_trans* tr for(rid = table_funcs.rids_next(rs); rid != oid_nil; rid = table_funcs.rids_next(rs)) { v = table_funcs.column_find_value(tr, tri_id, rid); list_append(dep_list, v); + // FIXME unchecked_malloc MNEW can return NULL v = MNEW(sht); *(sht*)v = depend_type; list_append(dep_list, v); @@ -191,6 +192,7 @@ sql_trans_schema_user_dependencies(sql_t for(rid = table_funcs.rids_next(users); rid != oid_nil; rid = table_funcs.rids_next(users)) { v = table_funcs.column_find_value(tr, auth_id, rid); list_append(l,v); + // FIXME unchecked_malloc MNEW can return NULL v = MNEW(sht); *(sht*)v = type; list_append(l,v); @@ -216,6 +218,7 @@ sql_trans_owner_schema_dependencies(sql_ for(rid = table_funcs.rids_next(rs); rid != oid_nil; rid = table_funcs.rids_next(rs)) { v = table_funcs.column_find_value(tr, schema_id, rid); list_append(l, v); + // FIXME unchecked_malloc MNEW can return NULL v = MNEW(sht); *(sht*)v = type; list_append(l,v); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Dec2016 - disabled r tests for now because of their hig...
Changeset: e798e9acbc37 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e798e9acbc37 Modified Files: clients/R/Tests/All Branch: Dec2016 Log Message: disabled r tests for now because of their high volatility diffs (20 lines): diff --git a/clients/R/Tests/All b/clients/R/Tests/All --- a/clients/R/Tests/All +++ b/clients/R/Tests/All @@ -1,8 +1,8 @@ -HAVE_LIBR?deps-install -HAVE_LIBR?install -HAVE_LIBR?deps-test -HAVE_LIBR?dbi -HAVE_LIBR?dplyr -HAVE_LIBR?dplyr-flights -HAVE_LIBR?copy_into_fwf -HAVE_LIBR&NOT_WIN32?dbapply +#HAVE_LIBR?deps-install +#HAVE_LIBR?install +#HAVE_LIBR?deps-test +#HAVE_LIBR?dbi +#HAVE_LIBR?dplyr +#HAVE_LIBR?dplyr-flights +#HAVE_LIBR?copy_into_fwf +#HAVE_LIBR&NOT_WIN32?dbapply ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jul2017 - making r tests run again
Changeset: 90e068ebfcf6 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90e068ebfcf6 Modified Files: clients/R/Tests/All clients/R/Tests/copy_into_fwf.stable.out clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.err clients/R/Tests/deps-install.R clients/R/Tests/deps-test.R clients/R/Tests/dplyr-flights.R clients/R/Tests/dplyr-flights.reqtests clients/R/Tests/dplyr.R clients/R/Tests/install.R Branch: Jul2017 Log Message: making r tests run again diffs (182 lines): diff --git a/clients/R/Tests/All b/clients/R/Tests/All --- a/clients/R/Tests/All +++ b/clients/R/Tests/All @@ -1,8 +1,8 @@ HAVE_LIBR?deps-install HAVE_LIBR?install HAVE_LIBR?deps-test -HAVE_LIBR&HAVE_PERL?dbi -HAVE_LIBR?dplyr +HAVE_LIBR?dbi +#HAVE_LIBR?dplyr too slow HAVE_LIBR?dplyr-flights HAVE_LIBR?copy_into_fwf HAVE_LIBR&NOT_WIN32?dbapply diff --git a/clients/R/Tests/copy_into_fwf.stable.out b/clients/R/Tests/copy_into_fwf.stable.out --- a/clients/R/Tests/copy_into_fwf.stable.out +++ b/clients/R/Tests/copy_into_fwf.stable.out @@ -26,6 +26,8 @@ Ready. SQL CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION) + ROWS Fetched: 0 [complete] + Changed: NA mpg cyl disp hp dratwt qsec vs am gear carb 1 21.0 6 160.0 110 3.90 2.620 16.46 0 144 2 21.0 6 160.0 110 3.90 2.875 17.02 0 144 diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R --- a/clients/R/Tests/dbi.R +++ b/clients/R/Tests/dbi.R @@ -198,7 +198,7 @@ stopifnot(identical(1L, as.integer(dbGet # reserved words in data frame column names stopifnot(dbIsValid(conn)) dbBegin(conn) -dbWriteTable(conn, "evilt", data.frame(year=42, month=12, day=24, some.dot=12), transaction=F) +dbWriteTable(conn, "evilt", data.frame(year=42, month=12, day=24, some.dot=12)) stopifnot(dbExistsTable(conn, "evilt")) dbRollback(conn) @@ -208,7 +208,7 @@ dbBegin(conn) data(api, package="survey") x <- apiclus1 x$idkey <- seq( nrow( x ) ) -dbWriteTable( conn , 'x' , x , transaction=F) +dbWriteTable( conn , 'x' , x ) stopifnot(dbExistsTable(conn, "x")) dbRollback(conn) @@ -217,7 +217,7 @@ stopifnot(!is.null(dbGetQuery(conn, "SEL #non-standard dbwritetable use dbBegin(conn) -dbWriteTable(conn, "vectable", 1:1000, transaction=F) +dbWriteTable(conn, "vectable", 1:1000) stopifnot(dbExistsTable(conn, "vectable")) dbRollback(conn) diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err --- a/clients/R/Tests/dbi.stable.err +++ b/clients/R/Tests/dbi.stable.err @@ -33,19 +33,16 @@ stderr of test 'dbi` in directory 'clien Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Warning message: 'MonetDBLite::isIdCurrent' is deprecated. -Use 'dbIsValid' instead. +Use 'use dbIsValid() instead' instead. See help("Deprecated") Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. -Treating character vector parameter as file name(s) for monetdb.read.csv() Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", "Species" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. -Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. -Identifier(s) "Need to quote this table name" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "some.dot" contain uppercase or reserved SQL characters and need(s) to be quoted in queries. Identifier(s) "year", "month", "day" are reserved SQL keywords and need(s) to be quoted in queries. Identifier(s) "sch.wide", "comp.imp", "yr.rnd", "acs.k3", "acs.46", "acs.core", "pct.resp", "not.h
MonetDB: Jul2017 - no warnings
Changeset: e5559912c6e8 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e5559912c6e8 Modified Files: clients/R/Tests/deps-install.R clients/R/Tests/install.R Branch: Jul2017 Log Message: no warnings diffs (17 lines): diff --git a/clients/R/Tests/deps-install.R b/clients/R/Tests/deps-install.R --- a/clients/R/Tests/deps-install.R +++ b/clients/R/Tests/deps-install.R @@ -1,4 +1,5 @@ # autoinstall DBI and digest, we need those to install MonetDB.R +options(warn = -1) packages_required <- c("digest", "DBI") cat("#~BeginProfilingOutput~#\n", file=stderr()) diff --git a/clients/R/Tests/install.R b/clients/R/Tests/install.R --- a/clients/R/Tests/install.R +++ b/clients/R/Tests/install.R @@ -1,3 +1,4 @@ +options(warn = -1) cat("#~BeginProfilingOutput~#\n", file=stderr()) cat("#~BeginProfilingOutput~#\n", file=stdout()) ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - R client tests update
Changeset: 151314949edf for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=151314949edf Modified Files: clients/R/Tests/deps-install.R clients/R/Tests/dplyr-flights.stable.out Branch: Jun2016 Log Message: R client tests update diffs (55 lines): diff --git a/clients/R/Tests/deps-install.R b/clients/R/Tests/deps-install.R --- a/clients/R/Tests/deps-install.R +++ b/clients/R/Tests/deps-install.R @@ -3,6 +3,9 @@ dir.create(unlist(strsplit(Sys.getenv("R # autoinstall DBI and digest, we need those to install MonetDB.R dd <- capture.output(suppressMessages(suppressWarnings({ - install.packages(c("devtools", "digest"), repos="http://cran.rstudio.com/";, quiet=T) + repos <- "http://cran.rstudio.com/"; + lp <- c("devtools", "digest") + install.packages(lp, repos=repos, quiet=T) + update.packages(repos=repos, ask=F, oldPkgs=lp) devtools::install_github("rstats-db/DBI", quiet=T) }))) diff --git a/clients/R/Tests/dplyr-flights.stable.out b/clients/R/Tests/dplyr-flights.stable.out --- a/clients/R/Tests/dplyr-flights.stable.out +++ b/clients/R/Tests/dplyr-flights.stable.out @@ -30,25 +30,25 @@ Ready. # 11:44:05 > [1] TRUE -[1] 336776 16 -[1] 42 16 -[1] 42 16 -[1] 42 16 -[1] 42 16 -[1] 42 16 +[1] 336776 19 +[1] 42 19 +[1] 42 19 +[1] 42 19 +[1] 42 19 +[1] 42 19 [1] 42 3 [1] 42 3 -[1] 42 13 -[1] 42 1 [1] 42 16 [1] 42 1 +[1] 42 19 +[1] 42 1 [1] 42 2 -[1] 42 18 -[1] 42 18 +[1] 42 21 +[1] 42 21 [1] 42 2 [1] 1 1 -[1] 10 16 -[1] 42 16 +[1] 10 19 +[1] 42 19 [1] 42 3 [1] 42 4 [1] 42 4 ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - R client, reflecting changes on package attac...
Changeset: 8eaea6e500ed for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8eaea6e500ed Modified Files: clients/R/Tests/All clients/R/Tests/dbapply.R clients/R/Tests/dbi.R clients/R/Tests/dplyr.R Branch: Jun2016 Log Message: R client, reflecting changes on package attachment diffs (44 lines): diff --git a/clients/R/Tests/All b/clients/R/Tests/All --- a/clients/R/Tests/All +++ b/clients/R/Tests/All @@ -1,6 +1,6 @@ HAVE_LIBR?deps-install +HAVE_LIBR?install HAVE_LIBR?deps-test -HAVE_LIBR?install HAVE_LIBR?dbi HAVE_LIBR?survey HAVE_LIBR?dplyr diff --git a/clients/R/Tests/dbapply.R b/clients/R/Tests/dbapply.R --- a/clients/R/Tests/dbapply.R +++ b/clients/R/Tests/dbapply.R @@ -1,6 +1,5 @@ -ll <- NULL if (Sys.getenv("TSTTRGDIR") != "") { - ll <- paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir") + .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir"))) } library(DBI, quietly = T) diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R --- a/clients/R/Tests/dbi.R +++ b/clients/R/Tests/dbi.R @@ -1,6 +1,5 @@ -ll <- NULL if (Sys.getenv("TSTTRGDIR") != "") { - ll <- paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir") + .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir"))) } library(DBI) diff --git a/clients/R/Tests/dplyr.R b/clients/R/Tests/dplyr.R --- a/clients/R/Tests/dplyr.R +++ b/clients/R/Tests/dplyr.R @@ -1,6 +1,5 @@ -ll <- NULL if (Sys.getenv("TSTTRGDIR") != "") { - ll <- paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir") + .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir"))) } ff <- textConnection("asdf", open="w") # hide output from connect and attach since it would blow up the test output ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - fixed a rather bizarre bug where the optimize...
Changeset: 49a7b5ffcfab for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=49a7b5ffcfab Modified Files: monetdb5/optimizer/opt_costModel.c sql/server/sql_parser.y Branch: Jun2016 Log Message: fixed a rather bizarre bug where the optimizer would call log(0) which would set errno which would never get cleared which would break a subsequent query with LIMIT 1 at the end. diffs (37 lines): diff --git a/monetdb5/optimizer/opt_costModel.c b/monetdb5/optimizer/opt_costModel.c --- a/monetdb5/optimizer/opt_costModel.c +++ b/monetdb5/optimizer/opt_costModel.c @@ -67,6 +67,10 @@ OPTcostModelImplementation(Client cntxt, } else if (getFunctionId(p) == crossRef) { newRows(1,2,((log((double) c1) + log((double) c2) > log(INT_MAX) ? INT_MAX : c1 * c2 +1)),0); + /* log sets errno if it cannot compute the log. This will then screw with code that checks errno */ + if (errno == ERANGE || errno == EDOM) { + errno = 0; + } } } else if (getModuleId(p) == batcalcRef) { if( getFunctionId(p) == ifthenelseRef) { diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y --- a/sql/server/sql_parser.y +++ b/sql/server/sql_parser.y @@ -5160,7 +5160,8 @@ lngval: { char *end = NULL, *s = $1; int l = _strlen(s); - + // errno might be non-zero due to other people's code + errno = 0; if (l <= 19) { $$ = strtoll(s,&end,10); } else { @@ -5182,7 +5183,8 @@ intval: { char *end = NULL, *s = $1; int l = _strlen(s); - + // errno might be non-zero due to other people's code + errno = 0; if (l <= 10) { $$ = strtol(s,&end,10); } else { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - Removed R client and embedded folder, updated...
Changeset: e7298bd42967 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e7298bd42967 Added Files: clients/R/Tests/install.timeout Removed Files: clients/R/MonetDB.R/DESCRIPTION clients/R/MonetDB.R/NAMESPACE clients/R/MonetDB.R/NEWS clients/R/MonetDB.R/R/control.R clients/R/MonetDB.R/R/dbapply.R clients/R/MonetDB.R/R/dbi.R clients/R/MonetDB.R/R/dplyr.R clients/R/MonetDB.R/R/mapi.R clients/R/MonetDB.R/man/MonetDB.R.Rd clients/R/MonetDB.R/man/MonetDBLite.Rd clients/R/MonetDB.R/man/control.Rd clients/R/MonetDB.R/man/dbApply.Rd clients/R/MonetDB.R/man/dbSendUpdate.Rd clients/R/MonetDB.R/man/dbTransaction.Rd clients/R/MonetDB.R/man/mc.Rd clients/R/MonetDB.R/man/monetdb.read.csv.Rd clients/R/MonetDB.R/man/monetdbRtype.Rd clients/R/MonetDB.R/man/monetdb_queryinfo.Rd clients/R/MonetDB.R/man/monetdbd.liststatus.Rd clients/R/MonetDB.R/man/mq.Rd clients/R/MonetDB.R/man/sqlitecompat.Rd clients/R/MonetDB.R/man/src_monetdb.Rd clients/R/MonetDB.R/src/mapisplit-r.c clients/R/MonetDB.R/src/mapisplit.c clients/R/MonetDB.R/src/mapisplit.h clients/R/Tests/survey.R clients/R/Tests/survey.reqtests clients/R/Tests/survey.stable.err clients/R/Tests/survey.stable.out clients/R/build-for-cran.sh tools/embedded/Makefile.ag tools/embedded/Tests/All tools/embedded/Tests/dbi.R tools/embedded/Tests/dbi.reqtests tools/embedded/Tests/dbi.stable.err tools/embedded/Tests/dbi.stable.out tools/embedded/Tests/deps-test.R tools/embedded/Tests/deps-test.stable.err tools/embedded/Tests/deps-test.stable.out tools/embedded/Tests/deps-test.timeout tools/embedded/Tests/dplyr.R tools/embedded/Tests/install-binary.R tools/embedded/Tests/install-binary.stable.err tools/embedded/Tests/install-binary.stable.out tools/embedded/Tests/install-source.R tools/embedded/Tests/install-source.stable.err tools/embedded/Tests/install-source.stable.out tools/embedded/Tests/install-source.timeout tools/embedded/Tests/install.reqtests tools/embedded/Tests/lowlevel.R tools/embedded/Tests/lowlevel.reqtests tools/embedded/Tests/lowlevel.stable.err tools/embedded/Tests/lowlevel.stable.out tools/embedded/build-on-windows.bat tools/embedded/build-rpkg.sh tools/embedded/demo.c tools/embedded/embedded.c tools/embedded/embedded.h tools/embedded/embeddedr.c tools/embedded/embeddedr.h tools/embedded/inlined_scripts.c tools/embedded/inlined_scripts.py tools/embedded/inlined_scripts.sh tools/embedded/pcre-for-osx.h tools/embedded/pkg-excludes tools/embedded/rcheck.sh tools/embedded/rpackage/DESCRIPTION tools/embedded/rpackage/NAMESPACE tools/embedded/rpackage/R/monetdb.R tools/embedded/rpackage/configure tools/embedded/rpackage/configure.win tools/embedded/rpackage/man/embedded.Rd tools/embedded/undef.h tools/embedded/windows/libtool.sh tools/embedded/windows/pmc.sh tools/embedded/windows/sedscript.tpl Modified Files: clients/Makefile.ag clients/R/Tests/All clients/R/Tests/dbapply.R clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.err clients/R/Tests/deps-install.R clients/R/Tests/deps-test.R clients/R/Tests/dplyr-flights.R clients/R/Tests/dplyr.R clients/R/Tests/install.R clients/R/Tests/install.stable.out clients/R/Tests/testrandom.R tools/Makefile.ag Branch: Jun2016 Log Message: Removed R client and embedded folder, updated tests diffs (truncated from 7845 to 300 lines): diff --git a/clients/Makefile.ag b/clients/Makefile.ag --- a/clients/Makefile.ag +++ b/clients/Makefile.ag @@ -6,4 +6,4 @@ SUBDIRS = HAVE_MAPI?mapilib HAVE_MAPI?mapiclient HAVE_ODBC?odbc HAVE_PERL?perl php examples HAVE_PYTHON2?python2 HAVE_PYTHON3?python3 NATIVE_WIN32?NT -EXTRA_DIST_DIR = Tests R ruby +EXTRA_DIST_DIR = Tests ruby diff --git a/clients/R/MonetDB.R/DESCRIPTION b/clients/R/MonetDB.R/DESCRIPTION deleted file mode 100644 --- a/clients/R/MonetDB.R/DESCRIPTION +++ /dev/null @@ -1,15 +0,0 @@ -Package: MonetDB.R -Version: 1.0.2 -Title: Connect MonetDB to R -Authors@R: c(person("Hannes Muehleisen", role = c("aut", "cre"),email = "han...@cwi.nl"), - person("Anthony Damico", role = "aut"), - person("Thomas Lumley", role = "ctb")) -Author: Hannes Muehleisen [aut, cre], Anthony Damico [aut], Thomas Lumley [ctb] -Maintainer: Hannes Muehleisen -Imports: DBI (>= 0.3.1), digest (
MonetDB: k3match-stable - backport of k3m to jul2015 branch
Changeset: 76340e22631a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=76340e22631a Added Files: sql/backends/monet5/k3m/3dtree.c sql/backends/monet5/k3m/3dtree.h sql/backends/monet5/k3m/67_k3m.mal sql/backends/monet5/k3m/67_k3m.sql sql/backends/monet5/k3m/Makefile.ag sql/backends/monet5/k3m/Tests/All sql/backends/monet5/k3m/Tests/k3m.inserttree.sql sql/backends/monet5/k3m/Tests/k3m.inserttree.stable.err sql/backends/monet5/k3m/Tests/k3m.inserttree.stable.out sql/backends/monet5/k3m/Tests/k3m.singletable.sql sql/backends/monet5/k3m/Tests/k3m.singletable.stable.err sql/backends/monet5/k3m/Tests/k3m.singletable.stable.out sql/backends/monet5/k3m/k3m.c sql/backends/monet5/k3m/k3m.mal sql/backends/monet5/k3m/k3match.h sql/backends/monet5/k3m/median.c sql/backends/monet5/k3m/median.h sql/backends/monet5/k3m/point.c sql/backends/monet5/k3m/point.h Modified Files: configure.ag sql/backends/monet5/Makefile.ag Branch: k3match-stable Log Message: backport of k3m to jul2015 branch diffs (truncated from 2519 to 300 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -2191,6 +2191,17 @@ if test "x$have_valgrind" != xno; then [if test "x$have_valgrind" = xyes; then AC_MSG_ERROR([no valgrind support found]); fi]) fi +org_have_k3m=no +have_k3m=$org_have_k3m +AC_ARG_WITH(k3m, + AS_HELP_STRING([--with-k3m], + [include Pim Schellart's K3Match library (default=no)]), + have_k3m=$withval) +if test "x$have_k3m" != xno; then + AC_DEFINE(HAVE_K3M, 1, [Define if you want to use the K3Match library]) +fi +AM_CONDITIONAL(HAVE_K3M, test x"$have_k3m" != xno) + # check for sphinxclient org_have_sphinxclient="auto" have_sphinxclient=$org_have_sphinxclient @@ -3336,6 +3347,7 @@ for comp in \ 'java ' \ 'java_control ' \ 'java_jdbc' \ + 'k3m ' \ 'libxml2 ' \ 'openssl ' \ 'pcre ' \ diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag --- a/sql/backends/monet5/Makefile.ag +++ b/sql/backends/monet5/Makefile.ag @@ -4,7 +4,7 @@ # # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. -SUBDIRS = NOT_WIN32?vaults UDF HAVE_SAMTOOLS?bam LSST HAVE_GSL?gsl generator +SUBDIRS = NOT_WIN32?vaults UDF HAVE_SAMTOOLS?bam LSST HAVE_GSL?gsl generator HAVE_K3M?k3m INCLUDES = ../../include ../../common ../../storage ../../server \ ../../../monetdb5/modules/atoms \ diff --git a/sql/backends/monet5/k3m/3dtree.c b/sql/backends/monet5/k3m/3dtree.c new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/k3m/3dtree.c @@ -0,0 +1,255 @@ +/** + * This file is part of K3Match. * + * Copyright (C) 2016 Pim Schellart * + ** + * This Source Code Form is subject to the terms of the Mozilla Public* + * License, v. 2.0. If a copy of the MPL was not distributed with this* + * file, You can obtain one at http://mozilla.org/MPL/2.0/. * + **/ + +#include +#include + +#include + +#define SQUARE(x) ((x) * (x)) + +void k3m_build_balanced_tree(node_t *tree, point_t **points, int_t npoints, int axis, int_t *npool) +{ + node_t *current = tree+(*npool); + + int next_axis = (axis + 1) % 3; + + int_t nleft, nright; + + current->left = NULL; + current->right = NULL; + current->axis = axis; + + current->point = k3m_median(points, npoints, current->axis); + + nright = npoints / 2; + nleft = nright - (1 - npoints % 2); + + if (nleft > 0) + { +(*npool)++; +current->left = tree+(*npool); +current->left->parent = &(*current); +k3m_build_balanced_tree(tree, points, nleft, next_axis, npool); + } + + if (nright > 0) + { +(*npool)++; +current->right = tree+(*npool); +current->right->parent = &(*current); +k3m_build_balanced_tree(tree, points+nleft+1, nright, next_axis, npool); + } +} + +node_t* k3m_insert_node(node_t *tree, node_t *node) +{ +int axis = 0; +node_t *parent = NULL; +node_t *current = tree; + +node->left = NULL; +node->right = NULL; +node->parent = NULL; + +while (current != NULL) { +parent = current; +if (node->point->value[axis] < current->point->value[axis]) { +current = current->left; +} else { +current = current->right; +} +axis = (axis + 1) % 3; +} + +node->parent = parent; +node->axis = axis; + +if (tree == NULL) { +tree = node; +} else if (node->point->value[parent->axis] < p
MonetDB: k3match-stable - getting this to run again
Changeset: 5490fa691955 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5490fa691955 Modified Files: sql/backends/monet5/k3m/k3m.c Branch: k3match-stable Log Message: getting this to run again diffs (50 lines): diff --git a/sql/backends/monet5/k3m/k3m.c b/sql/backends/monet5/k3m/k3m.c --- a/sql/backends/monet5/k3m/k3m.c +++ b/sql/backends/monet5/k3m/k3m.c @@ -63,7 +63,7 @@ str K3Mbuild(Client cntxt, MalBlkPtr mb, bit newtree = !k3m_tree; k3m_tree_tpe *k3m_tree_alloc = NULL; (void) cntxt; - MT_lock_set(&k3m_lock); + MT_lock_set(&k3m_lock, "K3Mbuild"); if (!isaBatType(getArgType(mb,pci,0)) || !isaBatType(getArgType(mb,pci,1)) || !isaBatType(getArgType(mb,pci,2)) || !isaBatType(getArgType(mb,pci,3))) { @@ -158,7 +158,7 @@ str K3Mbuild(Client cntxt, MalBlkPtr mb, k3m_tree->tree = k3m_insert_node(k3m_tree->tree, &k3m_tree_alloc->tree[i]); } } - MT_lock_unset(&k3m_lock); + MT_lock_unset(&k3m_lock, "K3Mbuild"); ret = BATnew(TYPE_void, TYPE_bit, 0, TRANSIENT); BUNappend(ret, &b, 0); *getArgReference_bat(stk, pci, 0) = ret->batCacheid; @@ -179,7 +179,7 @@ str K3Mfree(Client cntxt, MalBlkPtr mb, (void) stk; (void) pci; - MT_lock_set(&k3m_lock); + MT_lock_set(&k3m_lock, "K3Mfree"); for (i = 0; i < k3m_allocs_pos; i++) { GDKfree(k3m_allocs[i]->tree); @@ -192,7 +192,7 @@ str K3Mfree(Client cntxt, MalBlkPtr mb, k3m_allocs_size = K3M_ALLOCS_DEFAULT_SIZE; k3m_tree = NULL; - MT_lock_unset(&k3m_lock); + MT_lock_unset(&k3m_lock, "K3Mfree"); ret = BATnew(TYPE_void, TYPE_bit, 0, TRANSIENT); BUNappend(ret, &b, 0); @@ -266,6 +266,10 @@ str K3Mquery(Client cntxt, MalBlkPtr mb, } GDKfree(search.value); + out_id_cat->hseqbase = 0; + out_id_sl->hseqbase = 0; + out_dist->hseqbase = 0; + BBPkeepref(out_id_cat->batCacheid); BBPkeepref(out_id_sl->batCacheid); BBPkeepref(out_dist->batCacheid); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: k3match-stable - merge with jul2015 branch
Changeset: 52411c63c5af for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=52411c63c5af Added Files: clients/Tests/SQL-dump.SQL.py java/tests/Test_CisValid.java sql/jdbc/tests/Tests/Test_CisValid.SQL.bat sql/jdbc/tests/Tests/Test_CisValid.SQL.sh sql/jdbc/tests/Tests/Test_CisValid.stable.err sql/jdbc/tests/Tests/Test_CisValid.stable.out sql/test/BugTracker-2016/Tests/NULLtinyintBAT.bin sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_ALTER_TABLE_crash.Bug-4010.sql sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_ALTER_TABLE_crash.Bug-4010.stable.err sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_ALTER_TABLE_crash.Bug-4010.stable.out sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_UPDATE_crash.Bug-4010.sql sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_UPDATE_crash.Bug-4010.stable.err sql/test/BugTracker-2016/Tests/RELEASE_SAVEPOINT_after_UPDATE_crash.Bug-4010.stable.out sql/test/BugTracker-2016/Tests/case-with-string-Bug.3932.sql sql/test/BugTracker-2016/Tests/case-with-string-Bug.3932.stable.err sql/test/BugTracker-2016/Tests/case-with-string-Bug.3932.stable.out sql/test/BugTracker-2016/Tests/column_name_conflict.Bug-3912.sql sql/test/BugTracker-2016/Tests/column_name_conflict.Bug-3912.stable.err sql/test/BugTracker-2016/Tests/column_name_conflict.Bug-3912.stable.out sql/test/BugTracker-2016/Tests/copy_into_with_header.Bug-3910.sql sql/test/BugTracker-2016/Tests/copy_into_with_header.Bug-3910.stable.err sql/test/BugTracker-2016/Tests/copy_into_with_header.Bug-3910.stable.out sql/test/BugTracker-2016/Tests/data-uuid.csv.bz2 sql/test/BugTracker-2016/Tests/data3987.csv sql/test/BugTracker-2016/Tests/date_calculation_and_comparison_wrong_results.Bug-3940.sql sql/test/BugTracker-2016/Tests/date_calculation_and_comparison_wrong_results.Bug-3940.stable.err sql/test/BugTracker-2016/Tests/date_calculation_and_comparison_wrong_results.Bug-3940.stable.out sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.sql sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.err sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out sql/test/BugTracker-2016/Tests/decimal_vs_integer.Bug-3941.stable.out.32bit sql/test/BugTracker-2016/Tests/epoch.Bug-3979.sql sql/test/BugTracker-2016/Tests/epoch.Bug-3979.stable.err sql/test/BugTracker-2016/Tests/epoch.Bug-3979.stable.out sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.sql sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.stable.err sql/test/BugTracker-2016/Tests/fk-smaller-pk.Bug-3983.stable.out sql/test/BugTracker-2016/Tests/foreign_key.Bug-3938.sql sql/test/BugTracker-2016/Tests/foreign_key.Bug-3938.stable.err sql/test/BugTracker-2016/Tests/foreign_key.Bug-3938.stable.out sql/test/BugTracker-2016/Tests/isaUUID_function.Bug-3997.sql sql/test/BugTracker-2016/Tests/isaUUID_function.Bug-3997.stable.err sql/test/BugTracker-2016/Tests/isaUUID_function.Bug-3997.stable.out sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.sql sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.stable.err sql/test/BugTracker-2016/Tests/join-with-references-2sides-crashes.Bug-3980.stable.out sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.sql sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.stable.err sql/test/BugTracker-2016/Tests/leftjoin.Bug-3981.stable.out sql/test/BugTracker-2016/Tests/malformed-copy-int.Bug-3987.sql.in sql/test/BugTracker-2016/Tests/malformed-copy-int.Bug-3987.stable.err sql/test/BugTracker-2016/Tests/malformed-copy-int.Bug-3987.stable.out sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.sql sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.stable.err sql/test/BugTracker-2016/Tests/memory-consumption-query-PLAN-25joins.Bug-3972.stable.out sql/test/BugTracker-2016/Tests/merge_project.Bug-3955.sql sql/test/BugTracker-2016/Tests/merge_project.Bug-3955.stable.err sql/test/BugTracker-2016/Tests/merge_project.Bug-3955.stable.out sql/test/BugTracker-2016/Tests/min-max-uuid-column-wrong-results.Bug-3953.sql.in sql/test/BugTracker-2016/Tests/min-max-uuid-column-wrong-results.Bug-3953.stable.err sql/test/BugTracker-2016/Tests/min-max-uuid-column-wrong-results.Bug-3953.stable.out sql/test/BugTracker-2016/Tests/noNULLtinyintBAT.bin sql/test/BugTracker-2016/Tests/null_in_null.Bug-3900.sql sql/test/BugTracker-2016/Tests/null_in_null.Bug-3900.stable.err sql/test/BugTr
MonetDB: default - Infrastructure for loopback queries from embe...
Changeset: dcd358d882e1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dcd358d882e1 Added Files: monetdb5/extras/rapi/Tests/rapi10.malC Modified Files: monetdb5/extras/rapi/Tests/All monetdb5/extras/rapi/rapi.R monetdb5/extras/rapi/rapi.c monetdb5/extras/rapi/rapi.h Branch: default Log Message: Infrastructure for loopback queries from embedded R functions diffs (74 lines): diff --git a/monetdb5/extras/rapi/Tests/All b/monetdb5/extras/rapi/Tests/All --- a/monetdb5/extras/rapi/Tests/All +++ b/monetdb5/extras/rapi/Tests/All @@ -7,3 +7,4 @@ HAVE_LIBR&NOT_WIN32?rapi05 HAVE_LIBR&NOT_WIN32?rapi06 HAVE_LIBR&NOT_WIN32?rapi07 HAVE_LIBR&NOT_WIN32?rapi08 +HAVE_LIBR&NOT_WIN32?rapi10 diff --git a/monetdb5/extras/rapi/Tests/rapi10.malC b/monetdb5/extras/rapi/Tests/rapi10.malC new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.malC @@ -0,0 +1,3 @@ + +r:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query('select * from tables;')"); +io.print(r); diff --git a/monetdb5/extras/rapi/rapi.R b/monetdb5/extras/rapi/rapi.R --- a/monetdb5/extras/rapi/rapi.R +++ b/monetdb5/extras/rapi/rapi.R @@ -51,3 +51,8 @@ rewireFunc("q", quit, "base") #rewireFunc("system2", system, "base") rm(rewireFunc) + +loopback_query <- function(query) { + dyn.load(file.path(MONETDB_BINDIR, "..", "lib", "monetdb5", "lib_rapi.so")) + .Call("RAPIloopback", query, package="lib_rapi") +} diff --git a/monetdb5/extras/rapi/rapi.c b/monetdb5/extras/rapi/rapi.c --- a/monetdb5/extras/rapi/rapi.c +++ b/monetdb5/extras/rapi/rapi.c @@ -271,7 +271,7 @@ str RAPIeval(Client cntxt, MalBlkPtr mb, // get the lock even before initialization of the R interpreter, as this can take a second and must be done only once. MT_lock_set(&rapiLock); - env = PROTECT(eval(lang1(install("new.env")),R_GlobalEnv)); + env = PROTECT(eval(lang1(install("new.env")), R_GlobalEnv)); assert(env != NULL); // first argument after the return contains the pointer to the sql_func structure @@ -444,6 +444,12 @@ str RAPIeval(Client cntxt, MalBlkPtr mb, return msg; } +void* RAPIloopback(void *query) { + //FIXME actually do something + return (void*) ScalarString(mkCharCE((char*)CHAR(STRING_ELT((SEXP) query, 0)), CE_UTF8)); +} + + str RAPIprelude(void *ret) { (void) ret; MT_lock_init(&rapiLock, "rapi_lock"); @@ -458,6 +464,8 @@ str RAPIprelude(void *ret) { throw(MAL, "rapi.eval", "failed to initialise R environment (%s)", initstatus); } + Rf_defineVar(Rf_install("MONETDB_BINDIR"), ScalarString(mkCharCE(BINDIR, CE_UTF8)), R_GlobalEnv); + } MT_lock_unset(&rapiLock); printf("# MonetDB/R module loaded\n"); diff --git a/monetdb5/extras/rapi/rapi.h b/monetdb5/extras/rapi/rapi.h --- a/monetdb5/extras/rapi/rapi.h +++ b/monetdb5/extras/rapi/rapi.h @@ -35,7 +35,7 @@ rapi_export str RAPIevalStd(Client cntxt InstrPtr pci); rapi_export str RAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); - +rapi_export void* RAPIloopback(void *query); rapi_export str RAPIprelude(void *ret); rapi_export void writeConsoleEx(const char * buf, int buflen, int foo); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - rapi loopback functional
Changeset: 6ef63ca909ae for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6ef63ca909ae Added Files: monetdb5/extras/rapi/Tests/rapi10.stable.err monetdb5/extras/rapi/Tests/rapi10.stable.out Modified Files: monetdb5/extras/rapi/Makefile.ag monetdb5/extras/rapi/Tests/rapi10.malC monetdb5/extras/rapi/converters.c.h monetdb5/extras/rapi/rapi.R monetdb5/extras/rapi/rapi.c Branch: default Log Message: rapi loopback functional diffs (290 lines): diff --git a/monetdb5/extras/rapi/Makefile.ag b/monetdb5/extras/rapi/Makefile.ag --- a/monetdb5/extras/rapi/Makefile.ag +++ b/monetdb5/extras/rapi/Makefile.ag @@ -10,7 +10,13 @@ INCLUDES = \ ../../../gdk \ ../../mal \ ../../modules/kernel \ + ../../modules/atoms \ + ../../modules/mal \ + ../../../sql/common \ ../../../sql/include \ + ../../../sql/server \ + ../../../sql/storage \ + ../../../sql/backends/monet5 \ $(libr_CFLAGS) MTSAFE diff --git a/monetdb5/extras/rapi/Tests/rapi10.malC b/monetdb5/extras/rapi/Tests/rapi10.malC --- a/monetdb5/extras/rapi/Tests/rapi10.malC +++ b/monetdb5/extras/rapi/Tests/rapi10.malC @@ -1,3 +1,6 @@ +r1:bat[:oid,:int] := rapi.eval(nil:ptr,"loopback_query(\"select cast(42 as integer)\")[[1]]"); +io.print(r1); -r:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query('select * from tables;')"); -io.print(r); + +r2:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query(\"select 'a' as str union all select 'b' as str\")$str"); +io.print(r2); diff --git a/monetdb5/extras/rapi/Tests/rapi10.stable.err b/monetdb5/extras/rapi/Tests/rapi10.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.stable.err @@ -0,0 +1,36 @@ +stderr of test 'rapi10` in directory 'monetdb5/extras/rapi` itself: + + +# 10:06:54 > +# 10:06:54 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36192" "--set" "mapi_usock=/var/tmp/mtest-42962/.s.monetdb.36192" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:06:54 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 36192 +# cmdline opt mapi_usock = /var/tmp/mtest-42962/.s.monetdb.36192 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 10:06:55 > +# 10:06:55 > "mclient" "-lmal" "-ftest" "-Eutf-8" "--host=/var/tmp/mtest-42962" "--port=36192" +# 10:06:55 > + + +# 10:06:55 > +# 10:06:55 > "Done." +# 10:06:55 > + diff --git a/monetdb5/extras/rapi/Tests/rapi10.stable.out b/monetdb5/extras/rapi/Tests/rapi10.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.stable.out @@ -0,0 +1,80 @@ +stdout of test 'rapi10` in directory 'monetdb5/extras/rapi` itself: + + +# 10:06:54 > +# 10:06:54 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36192" "--set" "mapi_usock=/var/tmp/mtest-42962/.s.monetdb.36192" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:06:54 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_extras_rapi', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36192/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-42962/.s.monetdb.36192 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/R module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql script:
MonetDB: default - rapi loopback test
Changeset: 798ed548c61c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=798ed548c61c Added Files: sql/backends/monet5/Tests/rapi20.sql sql/backends/monet5/Tests/rapi20.stable.err sql/backends/monet5/Tests/rapi20.stable.out Modified Files: sql/backends/monet5/Tests/All Branch: default Log Message: rapi loopback test diffs (159 lines): diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All --- a/sql/backends/monet5/Tests/All +++ b/sql/backends/monet5/Tests/All @@ -19,6 +19,8 @@ HAVE_LIBR&NOT_WIN32?rapi16 HAVE_LIBR&NOT_WIN32?rapi17 HAVE_LIBR&NOT_WIN32?rapi18 HAVE_LIBR&NOT_WIN32?rapi19 +HAVE_LIBR&NOT_WIN32?rapi20 + HAVE_LIBPY?pyapi00 HAVE_LIBPY?pyapi01 diff --git a/sql/backends/monet5/Tests/rapi20.sql b/sql/backends/monet5/Tests/rapi20.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.sql @@ -0,0 +1,12 @@ +START TRANSACTION; + +CREATE TABLE rapi20dummy (i INTEGER); +INSERT INTO rapi20dummy VALUES (42), (84); + +CREATE FUNCTION rapi20() RETURNS TABLE(i INTEGER) LANGUAGE R { + loopback_query("SELECT * FROM rapi20dummy")$i +}; + +SELECT * FROM rapi20(); + +ROLLBACK; diff --git a/sql/backends/monet5/Tests/rapi20.stable.err b/sql/backends/monet5/Tests/rapi20.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.stable.err @@ -0,0 +1,36 @@ +stderr of test 'rapi20` in directory 'sql/backends/monet5` itself: + + +# 10:19:04 > +# 10:19:04 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30670" "--set" "mapi_usock=/var/tmp/mtest-43588/.s.monetdb.30670" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:19:04 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 30670 +# cmdline opt mapi_usock = /var/tmp/mtest-43588/.s.monetdb.30670 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5 +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 10:19:04 > +# 10:19:04 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-43588" "--port=30670" +# 10:19:04 > + + +# 10:19:05 > +# 10:19:05 > "Done." +# 10:19:05 > + diff --git a/sql/backends/monet5/Tests/rapi20.stable.out b/sql/backends/monet5/Tests/rapi20.stable.out new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.stable.out @@ -0,0 +1,84 @@ +stdout of test 'rapi20` in directory 'sql/backends/monet5` itself: + + +# 10:19:04 > +# 10:19:04 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30670" "--set" "mapi_usock=/var/tmp/mtest-43588/.s.monetdb.30670" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:19:04 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_sql_backends_monet5', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:30670/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-43588/.s.monetdb.30670 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/R module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql script: 20_vacuum.sql +# loading sql script: 21_dependency_functions.sql +# loading sql script: 22_clients.sql +# loading sql script: 23_skyserver.sql +# loading sql script: 24_zorder.sql +# loading sql script: 25_debug.sql +# loading sql script: 26_sysmon.sql +# loading sql script: 27_rejects.sql +# loading sql script: 39_analytics.sql +# loading sql script: 39_analytics_hge.sql +# loading sql scrip
MonetDB: Jun2016 - dbRemoveTable now throws error when the table...
Changeset: c14a081c5713 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c14a081c5713 Modified Files: clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.out Branch: Jun2016 Log Message: dbRemoveTable now throws error when the table does not exist diffs (22 lines): diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R --- a/clients/R/Tests/dbi.R +++ b/clients/R/Tests/dbi.R @@ -152,7 +152,6 @@ dbWriteTable(conn,tname,mtcars,append=T, stopifnot(identical(as.integer(2*nrow(mtcars)),tsize(conn,tname))) dbRemoveTable(conn,tname) -dbRemoveTable(conn,tname) dbWriteTable(conn,tname,mtcars,append=F,overwrite=F,insert=T) dbRemoveTable(conn,tname) diff --git a/clients/R/Tests/dbi.stable.out b/clients/R/Tests/dbi.stable.out --- a/clients/R/Tests/dbi.stable.out +++ b/clients/R/Tests/dbi.stable.out @@ -62,7 +62,6 @@ Ready. [1] TRUE [1] TRUE [1] TRUE -[1] FALSE [1] TRUE [1] TRUE [1] TRUE ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - Infrastructure for loopback queries from embe...
Changeset: f444d18b0e4c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f444d18b0e4c Added Files: monetdb5/extras/rapi/Tests/rapi10.malC Modified Files: monetdb5/extras/rapi/Tests/All monetdb5/extras/rapi/rapi.R monetdb5/extras/rapi/rapi.c monetdb5/extras/rapi/rapi.h Branch: Jun2016 Log Message: Infrastructure for loopback queries from embedded R functions diffs (74 lines): diff --git a/monetdb5/extras/rapi/Tests/All b/monetdb5/extras/rapi/Tests/All --- a/monetdb5/extras/rapi/Tests/All +++ b/monetdb5/extras/rapi/Tests/All @@ -7,3 +7,4 @@ HAVE_LIBR&NOT_WIN32?rapi05 HAVE_LIBR&NOT_WIN32?rapi06 HAVE_LIBR&NOT_WIN32?rapi07 HAVE_LIBR&NOT_WIN32?rapi08 +HAVE_LIBR&NOT_WIN32?rapi10 diff --git a/monetdb5/extras/rapi/Tests/rapi10.malC b/monetdb5/extras/rapi/Tests/rapi10.malC new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.malC @@ -0,0 +1,3 @@ + +r:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query('select * from tables;')"); +io.print(r); diff --git a/monetdb5/extras/rapi/rapi.R b/monetdb5/extras/rapi/rapi.R --- a/monetdb5/extras/rapi/rapi.R +++ b/monetdb5/extras/rapi/rapi.R @@ -51,3 +51,8 @@ rewireFunc("q", quit, "base") #rewireFunc("system2", system, "base") rm(rewireFunc) + +loopback_query <- function(query) { + dyn.load(file.path(MONETDB_BINDIR, "..", "lib", "monetdb5", "lib_rapi.so")) + .Call("RAPIloopback", query, package="lib_rapi") +} diff --git a/monetdb5/extras/rapi/rapi.c b/monetdb5/extras/rapi/rapi.c --- a/monetdb5/extras/rapi/rapi.c +++ b/monetdb5/extras/rapi/rapi.c @@ -271,7 +271,7 @@ str RAPIeval(Client cntxt, MalBlkPtr mb, // get the lock even before initialization of the R interpreter, as this can take a second and must be done only once. MT_lock_set(&rapiLock); - env = PROTECT(eval(lang1(install("new.env")),R_GlobalEnv)); + env = PROTECT(eval(lang1(install("new.env")), R_GlobalEnv)); assert(env != NULL); // first argument after the return contains the pointer to the sql_func structure @@ -444,6 +444,12 @@ str RAPIeval(Client cntxt, MalBlkPtr mb, return msg; } +void* RAPIloopback(void *query) { + //FIXME actually do something + return (void*) ScalarString(mkCharCE((char*)CHAR(STRING_ELT((SEXP) query, 0)), CE_UTF8)); +} + + str RAPIprelude(void *ret) { (void) ret; MT_lock_init(&rapiLock, "rapi_lock"); @@ -458,6 +464,8 @@ str RAPIprelude(void *ret) { throw(MAL, "rapi.eval", "failed to initialise R environment (%s)", initstatus); } + Rf_defineVar(Rf_install("MONETDB_BINDIR"), ScalarString(mkCharCE(BINDIR, CE_UTF8)), R_GlobalEnv); + } MT_lock_unset(&rapiLock); printf("# MonetDB/R module loaded\n"); diff --git a/monetdb5/extras/rapi/rapi.h b/monetdb5/extras/rapi/rapi.h --- a/monetdb5/extras/rapi/rapi.h +++ b/monetdb5/extras/rapi/rapi.h @@ -35,7 +35,7 @@ rapi_export str RAPIevalStd(Client cntxt InstrPtr pci); rapi_export str RAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); - +rapi_export void* RAPIloopback(void *query); rapi_export str RAPIprelude(void *ret); rapi_export void writeConsoleEx(const char * buf, int buflen, int foo); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: Jun2016 - rapi loopback functional
Changeset: ca0a6dc1b758 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ca0a6dc1b758 Added Files: monetdb5/extras/rapi/Tests/rapi10.stable.err monetdb5/extras/rapi/Tests/rapi10.stable.out Modified Files: monetdb5/extras/rapi/Makefile.ag monetdb5/extras/rapi/Tests/rapi10.malC monetdb5/extras/rapi/converters.c.h monetdb5/extras/rapi/rapi.R monetdb5/extras/rapi/rapi.c Branch: Jun2016 Log Message: rapi loopback functional diffs (290 lines): diff --git a/monetdb5/extras/rapi/Makefile.ag b/monetdb5/extras/rapi/Makefile.ag --- a/monetdb5/extras/rapi/Makefile.ag +++ b/monetdb5/extras/rapi/Makefile.ag @@ -10,7 +10,13 @@ INCLUDES = \ ../../../gdk \ ../../mal \ ../../modules/kernel \ + ../../modules/atoms \ + ../../modules/mal \ + ../../../sql/common \ ../../../sql/include \ + ../../../sql/server \ + ../../../sql/storage \ + ../../../sql/backends/monet5 \ $(libr_CFLAGS) MTSAFE diff --git a/monetdb5/extras/rapi/Tests/rapi10.malC b/monetdb5/extras/rapi/Tests/rapi10.malC --- a/monetdb5/extras/rapi/Tests/rapi10.malC +++ b/monetdb5/extras/rapi/Tests/rapi10.malC @@ -1,3 +1,6 @@ +r1:bat[:oid,:int] := rapi.eval(nil:ptr,"loopback_query(\"select cast(42 as integer)\")[[1]]"); +io.print(r1); -r:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query('select * from tables;')"); -io.print(r); + +r2:bat[:oid,:str] := rapi.eval(nil:ptr,"loopback_query(\"select 'a' as str union all select 'b' as str\")$str"); +io.print(r2); diff --git a/monetdb5/extras/rapi/Tests/rapi10.stable.err b/monetdb5/extras/rapi/Tests/rapi10.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.stable.err @@ -0,0 +1,36 @@ +stderr of test 'rapi10` in directory 'monetdb5/extras/rapi` itself: + + +# 10:06:54 > +# 10:06:54 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36192" "--set" "mapi_usock=/var/tmp/mtest-42962/.s.monetdb.36192" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:06:54 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 36192 +# cmdline opt mapi_usock = /var/tmp/mtest-42962/.s.monetdb.36192 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 10:06:55 > +# 10:06:55 > "mclient" "-lmal" "-ftest" "-Eutf-8" "--host=/var/tmp/mtest-42962" "--port=36192" +# 10:06:55 > + + +# 10:06:55 > +# 10:06:55 > "Done." +# 10:06:55 > + diff --git a/monetdb5/extras/rapi/Tests/rapi10.stable.out b/monetdb5/extras/rapi/Tests/rapi10.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/extras/rapi/Tests/rapi10.stable.out @@ -0,0 +1,80 @@ +stdout of test 'rapi10` in directory 'monetdb5/extras/rapi` itself: + + +# 10:06:54 > +# 10:06:54 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36192" "--set" "mapi_usock=/var/tmp/mtest-42962/.s.monetdb.36192" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_monetdb5_extras_rapi" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:06:54 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_extras_rapi', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36192/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-42962/.s.monetdb.36192 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/R module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql script:
MonetDB: Jun2016 - rapi loopback test
Changeset: a17d2377c16a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a17d2377c16a Added Files: sql/backends/monet5/Tests/rapi20.sql sql/backends/monet5/Tests/rapi20.stable.err sql/backends/monet5/Tests/rapi20.stable.out Modified Files: sql/backends/monet5/Tests/All Branch: Jun2016 Log Message: rapi loopback test diffs (159 lines): diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All --- a/sql/backends/monet5/Tests/All +++ b/sql/backends/monet5/Tests/All @@ -19,6 +19,8 @@ HAVE_LIBR&NOT_WIN32?rapi16 HAVE_LIBR&NOT_WIN32?rapi17 HAVE_LIBR&NOT_WIN32?rapi18 HAVE_LIBR&NOT_WIN32?rapi19 +HAVE_LIBR&NOT_WIN32?rapi20 + # should this work? #inlineUDF diff --git a/sql/backends/monet5/Tests/rapi20.sql b/sql/backends/monet5/Tests/rapi20.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.sql @@ -0,0 +1,12 @@ +START TRANSACTION; + +CREATE TABLE rapi20dummy (i INTEGER); +INSERT INTO rapi20dummy VALUES (42), (84); + +CREATE FUNCTION rapi20() RETURNS TABLE(i INTEGER) LANGUAGE R { + loopback_query("SELECT * FROM rapi20dummy")$i +}; + +SELECT * FROM rapi20(); + +ROLLBACK; diff --git a/sql/backends/monet5/Tests/rapi20.stable.err b/sql/backends/monet5/Tests/rapi20.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.stable.err @@ -0,0 +1,36 @@ +stderr of test 'rapi20` in directory 'sql/backends/monet5` itself: + + +# 10:19:04 > +# 10:19:04 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30670" "--set" "mapi_usock=/var/tmp/mtest-43588/.s.monetdb.30670" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:19:04 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 30670 +# cmdline opt mapi_usock = /var/tmp/mtest-43588/.s.monetdb.30670 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5 +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 10:19:04 > +# 10:19:04 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-43588" "--port=30670" +# 10:19:04 > + + +# 10:19:05 > +# 10:19:05 > "Done." +# 10:19:05 > + diff --git a/sql/backends/monet5/Tests/rapi20.stable.out b/sql/backends/monet5/Tests/rapi20.stable.out new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/rapi20.stable.out @@ -0,0 +1,84 @@ +stdout of test 'rapi20` in directory 'sql/backends/monet5` itself: + + +# 10:19:04 > +# 10:19:04 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30670" "--set" "mapi_usock=/var/tmp/mtest-43588/.s.monetdb.30670" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 10:19:04 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_sql_backends_monet5', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:30670/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-43588/.s.monetdb.30670 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/R module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql script: 20_vacuum.sql +# loading sql script: 21_dependency_functions.sql +# loading sql script: 22_clients.sql +# loading sql script: 23_skyserver.sql +# loading sql script: 24_zorder.sql +# loading sql script: 25_debug.sql +# loading sql script: 26_sysmon.sql +# loading sql script: 27_rejects.sql +# loading sql script: 39_analytics.sql +# loading sql script: 39_analytics_hge.sql +# loading sql script: 40_g
MonetDB: fixed-width-format - First part of fixed-width-format i...
Changeset: e871ab80d0f5 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e871ab80d0f5 Added Files: sql/backends/monet5/Tests/fwf.malC sql/backends/monet5/fwf.c sql/backends/monet5/fwf.h sql/backends/monet5/fwf.mal Modified Files: sql/backends/monet5/Makefile.ag Branch: fixed-width-format Log Message: First part of fixed-width-format import diffs (248 lines): diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag --- a/sql/backends/monet5/Makefile.ag +++ b/sql/backends/monet5/Makefile.ag @@ -46,7 +46,8 @@ lib__sql = { sql_round.c sql_round_impl.h sql_bat2time.c \ sql_fround.c sql_fround_impl.h \ sql_orderidx.c sql_orderidx.h \ - sql_rank.c sql_rank.h + sql_rank.c sql_rank.h \ + fwf.c fwf.h LIBS = ../../server/libsqlserver \ ../../storage/libstore \ ../../storage/bat/libbatstore \ @@ -64,7 +65,8 @@ headers_mal = { DIR = libdir/monetdb5 SOURCES = sql_aggr_bte.mal sql_aggr_flt.mal sql_aggr_dbl.mal sql_aggr_int.mal sql_aggr_lng.mal \ sql_aggr_sht.mal sql_decimal.mal sql_inspect.mal \ - sql_rank.mal sql.mal + sql_rank.mal sql.mal \ + fwf.mal } headers_mal_hge = { diff --git a/sql/backends/monet5/Tests/fwf.malC b/sql/backends/monet5/Tests/fwf.malC new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/fwf.malC @@ -0,0 +1,25 @@ +# write.fwf(mtcars, "/tmp/mtcars", header=F) + +# CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION) + +#4 2 6 4 56 6 2 2 2 2 +# 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4 + +sql.start2(); + +bint:= bat.new(:oid,:int); +bat.append(bint,4:int); +bat.append(bint,2:int); +bat.append(bint,6:int); +bat.append(bint,4:int); +bat.append(bint,5:int); +bat.append(bint,6:int); +bat.append(bint,6:int); +bat.append(bint,2:int); +bat.append(bint,2:int); +bat.append(bint,2:int); +bat.append(bint,2:int); + +include fwf; + +fwf.load("sys", "mtcars", "/tmp/mtcars", bint, 32:bte); diff --git a/sql/backends/monet5/fwf.c b/sql/backends/monet5/fwf.c new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/fwf.c @@ -0,0 +1,143 @@ +/* Ghetto programming warning. */ + +#include "monetdb_config.h" +#include "mal.h" +#include "mal_stack.h" +#include "mal_linker.h" +#include "gdk_utils.h" +#include "gdk.h" +#include "mmath.h" +#include "sql_catalog.h" +#include "sql_execute.h" +#include "fwf.h" + +str fwf_load(mvc *m, char* schema, char* table, char* filename, BAT *widths, char padding) { + FILE *f; + size_t fsize; +struct stat stb; +char* fptr, *fcur = NULL; +size_t i; +size_t width_sum = 0; + sql_schema *s; + sql_table *t; + node *n; + str msg = MAL_SUCCEED; + size_t ncol; + size_t approx_nrows; + size_t row = 0; + BAT** appends; + char* line = NULL; + int *widths_ptr = (int*) widths->T->heap.base; + + s = mvc_bind_schema(m, schema); + if (s == NULL) + throw(SQL, "sql.append", "Schema missing"); + t = mvc_bind_table(m, s, table); + if (t == NULL) + throw(SQL, "sql.append", "Table missing"); + + f = fopen(filename, "r"); + if (fstat(fileno(f), &stb) != 0) { + msg = createException(MAL, "fwf.load", "Could not stat file"); + goto cleanup; +} +fsize = (size_t) stb.st_size; +fptr = GDKmmap(filename, MMAP_READ, fsize); + if (!fptr) { + msg = createException(MAL, "fwf.load", "Could not map file"); + goto cleanup; + } + + ncol = t->columns.set->cnt; + if (BATcount(widths) != ncol) { + msg = createException(MAL, "fwf.load", "Incorrect number of widths supplied"); + goto cleanup; + } + + appends = malloc(sizeof(BAT*)*ncol); + if (!appends) { + msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); + goto cleanup; + } + + for (i = 0; i < ncol; i++) { + width_sum += widths_ptr[i]; + } + line = GDKmalloc(width_sum + ncol + 1); + if (!line) { + msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); + goto cleanup; + } + + approx_nrows = fsize/width_sum; + i = 0; + for (n = t->columns.set->h; n; n = n->next) { + sql_column *c = n->data; + appends[i] = BATnew(TYPE_void, c->type.type->localtype, approx_nrows, TRANSIENT) ; + if (!appends[i]) { + msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); +
MonetDB: fixed-width-format - new fwf stream wrapper
Changeset: cd474532e378 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=cd474532e378 Modified Files: common/stream/stream.c common/stream/stream.h Branch: fixed-width-format Log Message: new fwf stream wrapper diffs (160 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4671,3 +4671,146 @@ stream * stream_blackhole_create (void) s->access = ST_WRITE; return s; } + + +/* fixed-width format streams */ + +#define STREAM_FWF_NAME "fwf" +#define STREAM_FWF_FIELD_SEP '|' +#define STREAM_FWF_ESCAPE '\\' +#define STREAM_FWF_RECORD_SEP '\n' + +typedef struct { + stream *s; + // config + size_t num_fields; + size_t *widths; + char filler; + // state + size_t line_len; + char* in_buf; + char* out_buf; + size_t out_buf_start; + size_t out_buf_remaining; +} stream_fwf_data; + + +static ssize_t +stream_fwf_read(stream *s, void *buf, size_t elmsize, size_t cnt) +{ + stream_fwf_data *fsd; + size_t to_write = cnt; + size_t buf_written = 0; + if (strcmp(s->name, STREAM_FWF_NAME) != 0 || elmsize != 1) { + return -1; + } + fsd = (stream_fwf_data*) s->stream_data.p; + + while (to_write > 0) { + // input conversion + if (fsd->out_buf_remaining == 0) { // need to convert next line + size_t field_idx, in_buf_pos = 0, out_buf_pos = 0; + ssize_t actually_read = fsd->s->read(fsd->s, fsd->in_buf, 1, fsd->line_len); + if (actually_read < (ssize_t) fsd->line_len) { // incomplete last line + if (actually_read < 0) { + return actually_read; // this is an error + } + return buf_written; // skip last line + } + for (field_idx = 0; field_idx < fsd->num_fields; field_idx++) { + char *val_start, *val_end; + val_start = fsd->in_buf + in_buf_pos; + in_buf_pos += fsd->widths[field_idx]; + val_end = fsd->in_buf + in_buf_pos - 1; + while (*val_start == fsd->filler) val_start++; + while (*val_end == fsd->filler) val_end--; + while (val_start <= val_end) { + if (*val_start == STREAM_FWF_FIELD_SEP) { + fsd->out_buf[out_buf_pos++] = STREAM_FWF_ESCAPE; + } + fsd->out_buf[out_buf_pos++] = *val_start++; + } + fsd->out_buf[out_buf_pos++] = STREAM_FWF_FIELD_SEP; + } + fsd->out_buf[out_buf_pos++] = STREAM_FWF_RECORD_SEP; + fsd->out_buf_remaining = out_buf_pos; + fsd->out_buf_start = 0; + } + + // now we know something is in output_buf so deliver it + if (fsd->out_buf_remaining <= to_write) { + memcpy((char*)buf + buf_written, fsd->out_buf + fsd->out_buf_start, fsd->out_buf_remaining); + to_write -= fsd->out_buf_remaining; + buf_written += fsd->out_buf_remaining; + fsd->out_buf_remaining = 0; + } else { + memcpy((char*) buf + buf_written, fsd->out_buf + fsd->out_buf_start, to_write); + fsd->out_buf_start += to_write; + fsd->out_buf_remaining -= to_write; + to_write = 0; + } + } + return cnt; +} + + +static void +stream_fwf_close(stream *s) +{ + if (strcmp(s->name, STREAM_FWF_NAME) == 0) { + stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p; + fsd->s->close(fsd->s); + free(fsd->widths); + free(fsd->in_buf); + free(fsd->out_buf); + free(fsd); + } + destroy(s); +} + +stream* +stream_fwf_create (stream *s, size_t num_fields, size_t *widths, char filler) +{ + stream *ns; + stream_fwf_data *fsd = malloc(sizeof(stream_fwf_data)); + size_t i, out_buf_len; + if (!fsd) { + return NULL; + } + fsd->s = s; + fsd->num_fields = num_fields; + fsd->widths = widths; + fsd->filler = filler; + fsd->line_len = 1; // newline + for (i = 0; i < num_fields; i++) { + fsd->line_len += widths[i]; + } + fsd->in_buf = malloc(fsd->line_len); + if (!fsd->in_buf) { + fr
MonetDB: fixed-width-format - small fix for return value
Changeset: d0d49b5d498b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d0d49b5d498b Modified Files: common/stream/stream.c Branch: fixed-width-format Log Message: small fix for return value diffs (16 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4747,10 +4747,11 @@ stream_fwf_read(stream *s, void *buf, si memcpy((char*) buf + buf_written, fsd->out_buf + fsd->out_buf_start, to_write); fsd->out_buf_start += to_write; fsd->out_buf_remaining -= to_write; + buf_written += to_write; to_write = 0; } } - return cnt; + return buf_written; } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: fixed-width-format - stream route is working
Changeset: 19b8af045b23 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19b8af045b23 Modified Files: common/stream/stream.c sql/backends/monet5/sql.c Branch: fixed-width-format Log Message: stream route is working diffs (38 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4760,13 +4760,13 @@ stream_fwf_close(stream *s) { if (strcmp(s->name, STREAM_FWF_NAME) == 0) { stream_fwf_data *fsd = (stream_fwf_data*) s->stream_data.p; - fsd->s->close(fsd->s); + mnstr_close(fsd->s); free(fsd->widths); free(fsd->in_buf); free(fsd->out_buf); free(fsd); } - destroy(s); + // FIXME destroy(s); } stream* diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -3483,6 +3483,15 @@ mvc_import_table_wrap(Client cntxt, MalB return msg; } GDKfree(fn); + + // FIXME + if (GDKgetenv_isyes("testfwf")) { + fprintf(stderr, "### FWF IMPORT ON %s ###\n", *fname); + size_t *widths = malloc(sizeof(size_t)*400); + size_t i = 0; + widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 12; widths[i++] = 8; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 8; widths[i++] = 8; widths[i++] = 30; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 4; widths[i++] = 4; widths[i++] = 2; widths[i++] = 8; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; wi dths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4; widths[i++] = 2; widths[i++] = 8; widths[i++] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i+ +] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 8; widths[i++] = 8; widths[i++] = 8; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++]
MonetDB: fixed-width-format - COPY INTO table FWF (1,2,3); now w...
Changeset: eb9a9582ba77 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=eb9a9582ba77 Modified Files: sql/backends/monet5/sql.c sql/common/sql_types.c sql/server/rel_updates.c sql/server/sql_parser.y sql/server/sql_scan.c Branch: fixed-width-format Log Message: COPY INTO table FWF (1,2,3); now works diffs (267 lines): diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -3410,10 +3410,16 @@ mvc_import_table_wrap(Client cntxt, MalB lng *offset = getArgReference_lng(stk, pci, pci->retc + 7); int *locked = getArgReference_int(stk, pci, pci->retc + 8); int *besteffort = getArgReference_int(stk, pci, pci->retc + 9); + char *fixed_widths = NULL; str msg = MAL_SUCCEED; bstream *s = NULL; stream *ss; + if (pci->argc - pci->retc > 10) { + fixed_widths = *getArgReference_str(stk, pci, pci->retc + 10); + + } + (void) mb; /* NOT USED */ if ((msg = checkSQLContext(cntxt)) != NULL) return msg; @@ -3483,14 +3489,29 @@ mvc_import_table_wrap(Client cntxt, MalB return msg; } GDKfree(fn); - - // FIXME - if (GDKgetenv_isyes("testfwf")) { - fprintf(stderr, "### FWF IMPORT ON %s ###\n", *fname); - size_t *widths = malloc(sizeof(size_t)*400); - size_t i = 0; - widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 12; widths[i++] = 8; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 8; widths[i++] = 8; widths[i++] = 30; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 4; widths[i++] = 4; widths[i++] = 2; widths[i++] = 8; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 9; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; wi dths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4; widths[i++] = 2; widths[i++] = 8; widths[i++] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i+ +] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 8; widths[i++] = 8; widths[i++] = 8; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
MonetDB: fixed-width-format - Remove first stab
Changeset: b88502716f21 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b88502716f21 Removed Files: sql/backends/monet5/Tests/fwf.malC sql/backends/monet5/fwf.c sql/backends/monet5/fwf.h sql/backends/monet5/fwf.mal Modified Files: sql/backends/monet5/Makefile.ag Branch: fixed-width-format Log Message: Remove first stab diffs (248 lines): diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag --- a/sql/backends/monet5/Makefile.ag +++ b/sql/backends/monet5/Makefile.ag @@ -46,8 +46,7 @@ lib__sql = { sql_round.c sql_round_impl.h sql_bat2time.c \ sql_fround.c sql_fround_impl.h \ sql_orderidx.c sql_orderidx.h \ - sql_rank.c sql_rank.h \ - fwf.c fwf.h + sql_rank.c sql_rank.h LIBS = ../../server/libsqlserver \ ../../storage/libstore \ ../../storage/bat/libbatstore \ @@ -65,8 +64,7 @@ headers_mal = { DIR = libdir/monetdb5 SOURCES = sql_aggr_bte.mal sql_aggr_flt.mal sql_aggr_dbl.mal sql_aggr_int.mal sql_aggr_lng.mal \ sql_aggr_sht.mal sql_decimal.mal sql_inspect.mal \ - sql_rank.mal sql.mal \ - fwf.mal + sql_rank.mal sql.mal } headers_mal_hge = { diff --git a/sql/backends/monet5/Tests/fwf.malC b/sql/backends/monet5/Tests/fwf.malC deleted file mode 100644 --- a/sql/backends/monet5/Tests/fwf.malC +++ /dev/null @@ -1,25 +0,0 @@ -# write.fwf(mtcars, "/tmp/mtcars", header=F) - -# CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION) - -#4 2 6 4 56 6 2 2 2 2 -# 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4 - -sql.start2(); - -bint:= bat.new(:oid,:int); -bat.append(bint,4:int); -bat.append(bint,2:int); -bat.append(bint,6:int); -bat.append(bint,4:int); -bat.append(bint,5:int); -bat.append(bint,6:int); -bat.append(bint,6:int); -bat.append(bint,2:int); -bat.append(bint,2:int); -bat.append(bint,2:int); -bat.append(bint,2:int); - -include fwf; - -fwf.load("sys", "mtcars", "/tmp/mtcars", bint, 32:bte); diff --git a/sql/backends/monet5/fwf.c b/sql/backends/monet5/fwf.c deleted file mode 100644 --- a/sql/backends/monet5/fwf.c +++ /dev/null @@ -1,143 +0,0 @@ -/* Ghetto programming warning. */ - -#include "monetdb_config.h" -#include "mal.h" -#include "mal_stack.h" -#include "mal_linker.h" -#include "gdk_utils.h" -#include "gdk.h" -#include "mmath.h" -#include "sql_catalog.h" -#include "sql_execute.h" -#include "fwf.h" - -str fwf_load(mvc *m, char* schema, char* table, char* filename, BAT *widths, char padding) { - FILE *f; - size_t fsize; -struct stat stb; -char* fptr, *fcur = NULL; -size_t i; -size_t width_sum = 0; - sql_schema *s; - sql_table *t; - node *n; - str msg = MAL_SUCCEED; - size_t ncol; - size_t approx_nrows; - size_t row = 0; - BAT** appends; - char* line = NULL; - int *widths_ptr = (int*) widths->T->heap.base; - - s = mvc_bind_schema(m, schema); - if (s == NULL) - throw(SQL, "sql.append", "Schema missing"); - t = mvc_bind_table(m, s, table); - if (t == NULL) - throw(SQL, "sql.append", "Table missing"); - - f = fopen(filename, "r"); - if (fstat(fileno(f), &stb) != 0) { - msg = createException(MAL, "fwf.load", "Could not stat file"); - goto cleanup; -} -fsize = (size_t) stb.st_size; -fptr = GDKmmap(filename, MMAP_READ, fsize); - if (!fptr) { - msg = createException(MAL, "fwf.load", "Could not map file"); - goto cleanup; - } - - ncol = t->columns.set->cnt; - if (BATcount(widths) != ncol) { - msg = createException(MAL, "fwf.load", "Incorrect number of widths supplied"); - goto cleanup; - } - - appends = malloc(sizeof(BAT*)*ncol); - if (!appends) { - msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); - goto cleanup; - } - - for (i = 0; i < ncol; i++) { - width_sum += widths_ptr[i]; - } - line = GDKmalloc(width_sum + ncol + 1); - if (!line) { - msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); - goto cleanup; - } - - approx_nrows = fsize/width_sum; - i = 0; - for (n = t->columns.set->h; n; n = n->next) { - sql_column *c = n->data; - appends[i] = BATnew(TYPE_void, c->type.type->localtype, approx_nrows, TRANSIENT) ; - if (!appends[i]) { - msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL); -
MonetDB: fixed-width-format - merge with default
Changeset: 4defb6578c55 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4defb6578c55 Added Files: debian/libmonetdb-client8.install debian/libmonetdb-stream8.install debian/libmonetdb13.install sql/backends/monet5/Tests/pyapi29.sql sql/backends/monet5/Tests/pyapi29.stable.err sql/backends/monet5/Tests/pyapi29.stable.out sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.oid32 sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.oid32 Removed Files: buildtools/ChangeLog.Jul2015 clients/ChangeLog.Jul2015 clients/mapilib/ChangeLog.Jul2015 clients/python2/ChangeLog.Jul2015 clients/python3/ChangeLog.Jul2015 common/stream/ChangeLog.Jul2015 debian/libmonetdb-client7.install debian/libmonetdb-stream7.install debian/libmonetdb12.install gdk/ChangeLog.Jul2015 geom/ChangeLog.Jul2015 java/ChangeLog.Jul2015 monetdb5/ChangeLog.Jul2015 sql/ChangeLog.Jul2015 testing/ChangeLog.Jul2015 tools/merovingian/ChangeLog.Jul2015 Modified Files: .hgtags MonetDB.spec NT/installer32/MonetDB-ODBC-Installer.vdproj NT/installer32/MonetDB5-Geom-Module.vdproj NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB-ODBC-Installer.vdproj NT/installer64/MonetDB5-Geom-Module.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj NT/monetdb_config.h.in NT/rules.msc buildtools/ChangeLog-Archive buildtools/ChangeLog.Jun2016 buildtools/doc/windowsbuild.rst clients/ChangeLog-Archive clients/ChangeLog.Jun2016 clients/R/Tests/dbi.R clients/R/Tests/dbi.stable.out clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/exports.stable.out clients/mapilib/mapi.rc clients/odbc/driver/driver.rc clients/odbc/winsetup/setup.rc clients/python2/setup.py clients/python3/setup.py common/utils/msabaoth.c configure.ag debian/changelog debian/control debian/fix-deb.sh gdk/ChangeLog-Archive gdk/ChangeLog.Jun2016 gdk/gdk_bbp.c gdk/gdk_logger.c gdk/gdk_utils.c gdk/libbat.rc geom/ChangeLog-Archive geom/ChangeLog.Jun2016 java/ChangeLog-Archive java/ChangeLog.Jun2016 java/Makefile.ag java/build.properties java/pom.xml java/release.txt java/src/main/java/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java java/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java libversions monetdb5/ChangeLog-Archive monetdb5/ChangeLog.Jun2016 monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out monetdb5/extras/rapi/rapi.R monetdb5/extras/rapi/rapi.c monetdb5/mal/Tests/tst010.stable.err monetdb5/mal/mal.c monetdb5/mal/mal.h monetdb5/mal/mal_dataflow.c monetdb5/mal/mal_debugger.c monetdb5/mal/mal_debugger.h monetdb5/mal/mal_function.c monetdb5/mal/mal_function.h monetdb5/mal/mal_instruction.c monetdb5/mal/mal_instruction.h monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_profiler.c monetdb5/modules/atoms/json.c monetdb5/modules/mal/groupby.c monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mdb.h monetdb5/modules/mal/mdb.mal monetdb5/modules/mal/tablet.c monetdb5/optimizer/Tests/dataflow3.stable.out monetdb5/optimizer/Tests/inline02.stable.out monetdb5/optimizer/Tests/inline03.stable.out monetdb5/optimizer/Tests/inline04.stable.out monetdb5/optimizer/Tests/inline06.stable.out monetdb5/optimizer/opt_aliases.c monetdb5/optimizer/opt_candidates.c monetdb5/optimizer/opt_coercion.c monetdb5/optimizer/opt_commonTerms.c monetdb5/optimizer/opt_constants.c monetdb5/optimizer/opt_costModel.c monetdb5/optimizer/opt_dataflow.c monetdb5/optimizer/opt_deadcode.c monetdb5/optimizer/opt_evaluate.c monetdb5/optimizer/opt_factorize.c monetdb5/optimizer/opt_garbageCollector.c monetdb5/optimizer/opt_generator.c monetdb5/optimizer/opt_inline.c monetdb5/optimizer/opt_json.c monetdb5/optimizer/opt_matpack.c monetdb5/optimizer/opt_mergetable.c monetdb5/optimizer/opt_mitosis.c monetdb5/optimizer/opt_multiplex.c monetdb5/optimizer/opt_pipes.c monetdb5/optimizer/opt_profiler.c monetdb5/optimizer/opt_projectionpath.c monetdb5/optimizer/opt_pushselect.c monetdb5/optimizer/opt_querylog.c monetdb5/optimizer/opt_recycler.c monetdb5/optimizer/opt_r
MonetDB: fixed-width-format - Test case
Changeset: 24ecd74d3d62 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=24ecd74d3d62 Added Files: clients/R/Tests/copy_into_fwf.R clients/R/Tests/copy_into_fwf.reqtests Modified Files: clients/R/Tests/All clients/R/Tests/deps-test.R common/stream/stream.c Branch: fixed-width-format Log Message: Test case diffs (76 lines): diff --git a/clients/R/Tests/All b/clients/R/Tests/All --- a/clients/R/Tests/All +++ b/clients/R/Tests/All @@ -4,4 +4,5 @@ HAVE_LIBR?deps-test HAVE_LIBR?dbi HAVE_LIBR?dplyr HAVE_LIBR?dplyr-flights +HAVE_LIBR?copy_into_fwf HAVE_LIBR&NOT_WIN32?dbapply diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.R @@ -0,0 +1,34 @@ +if (Sys.getenv("TSTTRGDIR") != "") { + .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir"))) +} +library(DBI, quietly = T) + +args <- commandArgs(trailingOnly = TRUE) +dbport <- 5 +dbname <- "mTests_clients_R" +if (length(args) > 0) + dbport <- args[[1]] +if (length(args) > 1) + dbname <- args[[2]] + +con <- dbConnect(MonetDBLite::MonetDB(), port = dbport, dbname = dbname, wait = T) +stopifnot(dbIsValid(con)) + +tf <- tempfile() + +gdata::write.fwf(mtcars, tf, colnames = FALSE) + +if (dbExistsTable(con, "mtcars")) dbRemoveTable(con, "mtcars") + +dbBegin(con) +dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)") + +dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) + +print(dbReadTable(con, "mtcars")) + +stopifnot(nrow(dbReadTable(con, "mtcars")) < 1) + +dbRollback(con) + +print("SUCCESS") diff --git a/clients/R/Tests/copy_into_fwf.reqtests b/clients/R/Tests/copy_into_fwf.reqtests new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.reqtests @@ -0,0 +1,1 @@ +dbi diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R --- a/clients/R/Tests/deps-test.R +++ b/clients/R/Tests/deps-test.R @@ -4,5 +4,5 @@ dd <- capture.output(suppressMessages(su repos <- 'http://cran.rstudio.com/' if(length(np)) install.packages(np, repos=repos, quiet=T) update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T) - })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13')) + })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata')) }))) diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4791,7 +4791,7 @@ stream_fwf_create (stream *s, size_t num free(fsd); return NULL; } - out_buf_len = fsd->line_len * 2; // TODO: what if this is not enough? + out_buf_len = fsd->line_len * 3; fsd->out_buf = malloc(out_buf_len); if (!fsd->out_buf) { free(fsd->in_buf); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: fixed-width-format - fix and test case
Changeset: 3e166b600cad for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3e166b600cad Added Files: clients/R/Tests/copy_into_fwf.stable.err clients/R/Tests/copy_into_fwf.stable.out Modified Files: clients/R/Tests/copy_into_fwf.R clients/R/Tests/dbi.R sql/backends/monet5/sql.c Branch: fixed-width-format Log Message: fix and test case diffs (157 lines): diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R --- a/clients/R/Tests/copy_into_fwf.R +++ b/clients/R/Tests/copy_into_fwf.R @@ -23,11 +23,11 @@ if (dbExistsTable(con, "mtcars")) dbRemo dbBegin(con) dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)") -dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) +res <- dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) print(dbReadTable(con, "mtcars")) -stopifnot(nrow(dbReadTable(con, "mtcars")) < 1) +stopifnot(nrow(dbReadTable(con, "mtcars")) > 1) dbRollback(con) diff --git a/clients/R/Tests/copy_into_fwf.stable.err b/clients/R/Tests/copy_into_fwf.stable.err new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.stable.err @@ -0,0 +1,35 @@ +stderr of test 'copy_into_fwf` in directory 'clients/R` itself: + + +# 16:40:22 > +# 16:40:22 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30658" "--set" "mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" "embedded_r=yes" +# 16:40:22 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 30658 +# cmdline opt mapi_usock = /var/tmp/mtest-16034/.s.monetdb.30658 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_clients_R +# cmdline opt embedded_r = yes +# cmdline opt gdk_debug = 536870922 + +# 16:40:23 > +# 16:40:23 > "R" "--vanilla" "--slave" "--args" "30658" +# 16:40:23 > + + +# 16:40:23 > +# 16:40:23 > "Done." +# 16:40:23 > + diff --git a/clients/R/Tests/copy_into_fwf.stable.out b/clients/R/Tests/copy_into_fwf.stable.out new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.stable.out @@ -0,0 +1,71 @@ +stdout of test 'copy_into_fwf` in directory 'clients/R` itself: + + +# 16:40:22 > +# 16:40:22 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30658" "--set" "mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" "embedded_r=yes" +# 16:40:22 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_clients_R', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:30658/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-16034/.s.monetdb.30658 +# MonetDB/SQL module loaded +# MonetDB/R module loaded + +Ready. + +# 16:40:23 > +# 16:40:23 > "R" "--vanilla" "--slave" "--args" "30658" +# 16:40:23 > + +[1] TRUE + + SQL CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION) + ROWS Fetched: 0 [complete] + Changed: NA +mpg cyl disp hp dratwt qsec vs am gear carb +1 21.0 6 160.0 110 3.90 2.620 16.46 0 144 +2 21.0 6 160.0 110 3.90 2.875 17.02 0 144 +3 22.8 4 108.0 93 3.85 2.320 18.61 1 141 +4 21.4 6 258.0 110 3.08 3.215 19.44 1 031 +5 18.7 8 360.0 175 3.15 3.440 17.02 0 032 +6 18.1 6 225.0 105 2.76 3.460 20.22 1 031 +7 14.3 8 360.0 245 3.21 3.570 15.84 0 034 +8 24.4 4 146.7 62 3.69 3.190 20.00 1 042 +9 22.8 4 140.
MonetDB: default - PyAPI: Some startup fixes
Changeset: f5e70fd645ad for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f5e70fd645ad Modified Files: configure.ag monetdb5/extras/pyapi/pyapi.c Branch: default Log Message: PyAPI: Some startup fixes diffs (29 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -2359,7 +2359,7 @@ if test "x$enable_pyintegration" != xno; ;; esac if test "x$have_pyconfig" = x; then - AC_PATH_PROG(PYCMD,python-config,,$XPATH) + AC_PATH_PROG(PYCMD,python-config,no,$PATH) if test "x$PYCMD" = x; then if test "x$enable_pyintegration" = xyes; then AC_MSG_ERROR([python-config library required for Python integration support]) diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c --- a/monetdb5/extras/pyapi/pyapi.c +++ b/monetdb5/extras/pyapi/pyapi.c @@ -1620,10 +1620,10 @@ str MT_lock_set(&pyapiLock); if (!pyapiInitialized) { str msg = MAL_SUCCEED; -char* iar = NULL; Py_Initialize(); -PyRun_SimpleString("import numpy"); -import_array1(iar); +if (PyRun_SimpleString("import numpy") != 0 || _import_array() < 0) { +return PyError_CreateException("Failed to initialize embedded python", NULL); +} msg = _connection_init(); marshal_module = PyImport_Import(PyString_FromString("marshal")); if (marshal_module == NULL) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: fixed-width-format - Some cleanup before merge. Niels?
Changeset: 769cc2e46a29 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=769cc2e46a29 Modified Files: clients/R/Tests/copy_into_fwf.R common/stream/stream.c common/stream/stream.h sql/backends/monet5/sql.c sql/server/rel_updates.c Branch: fixed-width-format Log Message: Some cleanup before merge. Niels? diffs (90 lines): diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R --- a/clients/R/Tests/copy_into_fwf.R +++ b/clients/R/Tests/copy_into_fwf.R @@ -23,7 +23,8 @@ if (dbExistsTable(con, "mtcars")) dbRemo dbBegin(con) dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)") -res <- dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) +# delimiters are ineffective for fwf import just set them to make sure they dont break stuff +res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) print(dbReadTable(con, "mtcars")) diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4674,11 +4674,7 @@ stream * stream_blackhole_create (void) /* fixed-width format streams */ - -#define STREAM_FWF_NAME "fwf" -#define STREAM_FWF_FIELD_SEP '|' -#define STREAM_FWF_ESCAPE '\\' -#define STREAM_FWF_RECORD_SEP '\n' +#define STREAM_FWF_NAME "fwf_ftw" typedef struct { stream *s; diff --git a/common/stream/stream.h b/common/stream/stream.h --- a/common/stream/stream.h +++ b/common/stream/stream.h @@ -87,6 +87,12 @@ typedef __int128_t hge; #define ST_READ 0 #define ST_WRITE 1 +/* fwf gets turned into a csv with these parameters */ +#define STREAM_FWF_FIELD_SEP '|' +#define STREAM_FWF_ESCAPE '\\' +#define STREAM_FWF_RECORD_SEP '\n' +#define STREAM_FWF_FILLER ' ' + typedef struct stream stream; /* some os specific initialization */ diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c --- a/sql/backends/monet5/sql.c +++ b/sql/backends/monet5/sql.c @@ -3501,17 +3501,26 @@ mvc_import_table_wrap(Client cntxt, MalB } widths = malloc(sizeof(size_t) * ncol); if (!widths) { - // TODO: free other stuff + mnstr_destroy(ss); + GDKfree(tsep); + GDKfree(rsep); + GDKfree(ssep); + GDKfree(ns); throw(MAL, "sql.copy_from", MAL_MALLOC_FAIL); } for (i = 0; i < width_len; i++) { - if (fixed_widths[i] == '|') { + if (fixed_widths[i] == STREAM_FWF_FIELD_SEP) { fixed_widths[i] = '\0'; widths[current_width_entry++] = (size_t) atoll(val_start); val_start = fixed_widths + i + 1; } } - ss = stream_fwf_create(ss, ncol, widths, ' '); + /* overwrite other delimiters to the ones the FWF stream uses */ + sprintf((char*) tsep, "%c", STREAM_FWF_FIELD_SEP); + sprintf((char*) rsep, "%c", STREAM_FWF_RECORD_SEP); + sprintf((char*) ssep, ""); + + ss = stream_fwf_create(ss, ncol, widths, STREAM_FWF_FILLER); } #if SIZEOF_VOID_P == 4 s = bstream_create(ss, 0x2); diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c --- a/sql/server/rel_updates.c +++ b/sql/server/rel_updates.c @@ -1167,7 +1167,7 @@ rel_import(mvc *sql, sql_table *t, char } char* fwf_string_cur = fwf_string; for (dn = fwf_widths->h; dn; dn = dn->next) { - fwf_string_cur += sprintf(fwf_string_cur, LLFMT"|", dn->data.l_val); + fwf_string_cur += sprintf(fwf_string_cur, LLFMT"%c", dn->data.l_val, STREAM_FWF_FIELD_SEP); ncol++; } if(list_length(f->res) != ncol) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: fixed-width-format - merge with default
Changeset: 07b23de27a1d for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=07b23de27a1d Modified Files: configure.ag monetdb5/extras/pyapi/pyapi.c Branch: fixed-width-format Log Message: merge with default diffs (29 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -2359,7 +2359,7 @@ if test "x$enable_pyintegration" != xno; ;; esac if test "x$have_pyconfig" = x; then - AC_PATH_PROG(PYCMD,python-config,,$XPATH) + AC_PATH_PROG(PYCMD,python-config,no,$PATH) if test "x$PYCMD" = x; then if test "x$enable_pyintegration" = xyes; then AC_MSG_ERROR([python-config library required for Python integration support]) diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c --- a/monetdb5/extras/pyapi/pyapi.c +++ b/monetdb5/extras/pyapi/pyapi.c @@ -1620,10 +1620,10 @@ str MT_lock_set(&pyapiLock); if (!pyapiInitialized) { str msg = MAL_SUCCEED; -char* iar = NULL; Py_Initialize(); -PyRun_SimpleString("import numpy"); -import_array1(iar); +if (PyRun_SimpleString("import numpy") != 0 || _import_array() < 0) { +return PyError_CreateException("Failed to initialize embedded python", NULL); +} msg = _connection_init(); marshal_module = PyImport_Import(PyString_FromString("marshal")); if (marshal_module == NULL) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - COPY INTO from fixed width format files
Changeset: e1a82a5485de for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1a82a5485de Added Files: clients/R/Tests/copy_into_fwf.R clients/R/Tests/copy_into_fwf.reqtests clients/R/Tests/copy_into_fwf.stable.err clients/R/Tests/copy_into_fwf.stable.out Modified Files: clients/R/Tests/All clients/R/Tests/dbi.R clients/R/Tests/deps-test.R common/stream/stream.c common/stream/stream.h sql/backends/monet5/Makefile.ag sql/backends/monet5/sql.c sql/common/sql_types.c sql/server/rel_updates.c sql/server/sql_parser.y sql/server/sql_scan.c Branch: default Log Message: COPY INTO from fixed width format files diffs (truncated from 656 to 300 lines): diff --git a/clients/R/Tests/All b/clients/R/Tests/All --- a/clients/R/Tests/All +++ b/clients/R/Tests/All @@ -4,4 +4,5 @@ HAVE_LIBR?deps-test HAVE_LIBR?dbi HAVE_LIBR?dplyr HAVE_LIBR?dplyr-flights +HAVE_LIBR?copy_into_fwf HAVE_LIBR&NOT_WIN32?dbapply diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.R @@ -0,0 +1,35 @@ +if (Sys.getenv("TSTTRGDIR") != "") { + .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir"))) +} +library(DBI, quietly = T) + +args <- commandArgs(trailingOnly = TRUE) +dbport <- 5 +dbname <- "mTests_clients_R" +if (length(args) > 0) + dbport <- args[[1]] +if (length(args) > 1) + dbname <- args[[2]] + +con <- dbConnect(MonetDBLite::MonetDB(), port = dbport, dbname = dbname, wait = T) +stopifnot(dbIsValid(con)) + +tf <- tempfile() + +gdata::write.fwf(mtcars, tf, colnames = FALSE) + +if (dbExistsTable(con, "mtcars")) dbRemoveTable(con, "mtcars") + +dbBegin(con) +dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)") + +# delimiters are ineffective for fwf import just set them to make sure they dont break stuff +res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)")) + +print(dbReadTable(con, "mtcars")) + +stopifnot(nrow(dbReadTable(con, "mtcars")) > 1) + +dbRollback(con) + +print("SUCCESS") diff --git a/clients/R/Tests/copy_into_fwf.reqtests b/clients/R/Tests/copy_into_fwf.reqtests new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.reqtests @@ -0,0 +1,1 @@ +dbi diff --git a/clients/R/Tests/copy_into_fwf.stable.err b/clients/R/Tests/copy_into_fwf.stable.err new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.stable.err @@ -0,0 +1,35 @@ +stderr of test 'copy_into_fwf` in directory 'clients/R` itself: + + +# 16:40:22 > +# 16:40:22 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30658" "--set" "mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" "embedded_r=yes" +# 16:40:22 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 30658 +# cmdline opt mapi_usock = /var/tmp/mtest-16034/.s.monetdb.30658 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_clients_R +# cmdline opt embedded_r = yes +# cmdline opt gdk_debug = 536870922 + +# 16:40:23 > +# 16:40:23 > "R" "--vanilla" "--slave" "--args" "30658" +# 16:40:23 > + + +# 16:40:23 > +# 16:40:23 > "Done." +# 16:40:23 > + diff --git a/clients/R/Tests/copy_into_fwf.stable.out b/clients/R/Tests/copy_into_fwf.stable.out new file mode 100644 --- /dev/null +++ b/clients/R/Tests/copy_into_fwf.stable.out @@ -0,0 +1,71 @@ +stdout of test 'copy_into_fwf` in directory 'clients/R` itself: + + +# 16:40:22 > +# 16:40:22 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=30658" "--set" "mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" "embedded_r=yes" +# 16:40:22 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_clients_R', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available
MonetDB: default - Exports output
Changeset: b6b5baf0a8c3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b6b5baf0a8c3 Modified Files: clients/Tests/exports.stable.out Branch: default Log Message: Exports output diffs (11 lines): diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -2636,6 +2636,7 @@ stream *socket_rstream(SOCKET socket, co stream *socket_wastream(SOCKET socket, const char *name); stream *socket_wstream(SOCKET socket, const char *name); stream *stream_blackhole_create(void); +stream *stream_fwf_create(stream *s, size_t num_fields, size_t *widths, char filler); stream *udp_rastream(const char *hostname, int port, const char *name); stream *udp_wastream(const char *hostname, int port, const char *name); stream *wbstream(stream *s, size_t buflen); ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - fwf fix for windows newlines and general gunk...
Changeset: ee6e38863857 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ee6e38863857 Modified Files: common/stream/stream.c Branch: default Log Message: fwf fix for windows newlines and general gunk behind fwf area in line diffs (57 lines): diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -4688,6 +4688,7 @@ typedef struct { char* out_buf; size_t out_buf_start; size_t out_buf_remaining; + char* nl_buf; } stream_fwf_data; @@ -4713,6 +4714,9 @@ stream_fwf_read(stream *s, void *buf, si } return buf_written; // skip last line } + // consume to next newline + while (fsd->s->read(fsd->s, fsd->nl_buf, 1, 1) == 1 && *fsd->nl_buf != '\n'); + for (field_idx = 0; field_idx < fsd->num_fields; field_idx++) { char *val_start, *val_end; val_start = fsd->in_buf + in_buf_pos; @@ -4760,6 +4764,7 @@ stream_fwf_close(stream *s) free(fsd->widths); free(fsd->in_buf); free(fsd->out_buf); + free(fsd->nl_buf); free(fsd); } // FIXME destroy(s); @@ -4778,7 +4783,7 @@ stream_fwf_create (stream *s, size_t num fsd->num_fields = num_fields; fsd->widths = widths; fsd->filler = filler; - fsd->line_len = 1; // newline + fsd->line_len = 0; for (i = 0; i < num_fields; i++) { fsd->line_len += widths[i]; } @@ -4795,10 +4800,17 @@ stream_fwf_create (stream *s, size_t num return NULL; } fsd->out_buf_remaining = 0; - + fsd->nl_buf = malloc(1); + if (!fsd->nl_buf) { + free(fsd->in_buf); + free(fsd->out_buf); + free(fsd); + return NULL; + } if ((ns = create_stream(STREAM_FWF_NAME)) == NULL) { free(fsd->in_buf); free(fsd->out_buf); + free(fsd->nl_buf); free(fsd); return NULL; } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pythonloader - first stage COPY INTO sometable FROM LOA...
Changeset: d38a839b1d0a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d38a839b1d0a Modified Files: monetdb5/extras/pyapi/pyapi.c monetdb5/extras/pyapi/pyapi.h monetdb5/extras/pyapi/pyapi.mal sql/backends/monet5/sql_gencode.c sql/include/sql_catalog.h sql/server/rel_psm.c sql/server/rel_select.c sql/server/rel_semantic.c sql/server/rel_updates.c sql/server/sql_mvc.h sql/server/sql_parser.h sql/server/sql_parser.y sql/server/sql_scan.c Branch: pythonloader Log Message: first stage COPY INTO sometable FROM LOADER somepythonfunction(); diffs (truncated from 368 to 300 lines): diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c --- a/monetdb5/extras/pyapi/pyapi.c +++ b/monetdb5/extras/pyapi/pyapi.c @@ -418,6 +418,12 @@ PyAPIevalAggr(Client cntxt, MalBlkPtr mb } str +PyAPIevalLoader(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) +{ +return PyAPIeval(cntxt, mb, stk, pci, 0, 0); +} + +str PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { return PyAPIeval(cntxt, mb, stk, pci, 1, 1); diff --git a/monetdb5/extras/pyapi/pyapi.h b/monetdb5/extras/pyapi/pyapi.h --- a/monetdb5/extras/pyapi/pyapi.h +++ b/monetdb5/extras/pyapi/pyapi.h @@ -96,6 +96,7 @@ pyapi_export str PyAPIevalStd(Client cnt pyapi_export str PyAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); pyapi_export str PyAPIevalStdMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); pyapi_export str PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); +pyapi_export str PyAPIevalLoader(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); pyapi_export str PyAPIprelude(void *ret); diff --git a/monetdb5/extras/pyapi/pyapi.mal b/monetdb5/extras/pyapi/pyapi.mal --- a/monetdb5/extras/pyapi/pyapi.mal +++ b/monetdb5/extras/pyapi/pyapi.mal @@ -25,6 +25,10 @@ pattern eval_aggr(fptr:ptr,expr:str,arg: address PyAPIevalAggr comment "grouped aggregates through Python"; +pattern eval_loader(fptr:ptr,expr:str,arg:any...):any... +address PyAPIevalLoader +comment "loader functions through Python"; + # initializer code command prelude() :void address PyAPIprelude; pyapi.prelude(); @@ -44,6 +48,10 @@ pattern eval_aggr(fptr:ptr,expr:str,arg: address PyAPIevalAggr comment "grouped aggregates through Python"; +pattern eval_loader(fptr:ptr,expr:str,arg:any...):any... +address PyAPIevalLoader +comment "loader functions through Python"; + module pyapimap; # The generic Python interface diff --git a/sql/backends/monet5/sql_gencode.c b/sql/backends/monet5/sql_gencode.c --- a/sql/backends/monet5/sql_gencode.c +++ b/sql/backends/monet5/sql_gencode.c @@ -2135,7 +2135,7 @@ static int if (f->func->lang == FUNC_LANG_R || f->func->lang == FUNC_LANG_PY || f->func->lang == FUNC_LANG_MAP_PY) q = pushStr(mb, q, f->func->query); /* first dynamic output of copy* functions */ - if (f->func->type == F_UNION) + if (f->func->type == F_UNION || f->func->type == F_LOADER) q = table_func_create_result(mb, q, f->func, f->res); if (list_length(s->op1->op4.lval)) tpe = tail_type(s->op1->op4.lval->h->data); @@ -3038,6 +3038,10 @@ backend_create_py_func(backend *be, sql_ f->mod = "pyapi"; f->imp = "eval_aggr"; break; + case F_LOADER: + f->mod = "pyapi"; + f->imp = "eval_loader"; + break; case F_PROC: /* no output */ case F_FUNC: default: /* ie also F_FILT and F_UNION for now */ diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h --- a/sql/include/sql_catalog.h +++ b/sql/include/sql_catalog.h @@ -281,6 +281,7 @@ typedef struct sql_arg { #define F_FILT 4 #define F_UNION 5 #define F_ANALYTIC 6 +#define F_LOADER 7 #define IS_FUNC(f) (f->type == F_FUNC) #define IS_PROC(f) (f->type == F_PROC) @@ -288,6 +289,7 @@ typedef struct sql_arg { #define IS_FILT(f) (f->type == F_FILT) #define IS_UNION(f) (f->type == F_UNION) #define IS_ANALYTIC(f) (f->type == F_ANALYTIC) +#define IS_LOADER(f) (f->type == F_LOADER) #define FUNC_LANG_INT 0/* internal */ #define FUNC_LANG_MAL 1 /* create sql external mod.func */ diff --git a/sql/server/rel_psm.c b/sql/server/rel_psm.c --- a/sql/server/rel_psm.c +++ b/sql/server/rel_psm.c @@ -699,10 +699,12 @@ rel_create_func(mvc *sql, dlist *qname, char is_table = (res && res->token == SQL_TABLE); char is_aggr = (type == F_AGGR); char is_func = (type != F_PROC); - char *F = is_aggr?"AGGREGATE":(is_func?"FUNCTION":"PROCEDURE"); + char is_loader = (type != F_LOADER); + + char *F = is_loader?"LOADER":(is_aggr?"
MonetDB: pythonloader - test case
Changeset: 84b713b6251e for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=84b713b6251e Added Files: sql/backends/monet5/Tests/pyapi30.sql Branch: pythonloader Log Message: test case diffs (24 lines): diff --git a/sql/backends/monet5/Tests/pyapi30.sql b/sql/backends/monet5/Tests/pyapi30.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/pyapi30.sql @@ -0,0 +1,19 @@ + +START TRANSACTION; + +CREATE TABLE mytable(i DOUBLE, d STRING); + +CREATE LOADER myfunc() LANGUAGE PYTHON { + emit({'i':42,'d':'hello'}) +}; + +SELECT * FROM functions WHERE name='myfunc'; + +EXPLAIN COPY INTO mytable FROM LOADER myfunc(); + +COPY INTO mytable FROM LOADER myfunc(); + +DROP TABLE mytable; +DROP LOADER myfunc; + +ROLLBACK; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pythonloader - Support for int syntax 4E6 (= 4000000) i...
Changeset: 5d255bd86516 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d255bd86516 Added Files: sql/backends/monet5/Tests/int_notation_1e5.sql sql/backends/monet5/Tests/int_notation_1e5.stable.err sql/backends/monet5/Tests/int_notation_1e5.stable.out Modified Files: gdk/gdk_atoms.c sql/backends/monet5/Tests/All Branch: pythonloader Log Message: Support for int syntax 4E6 (= 400) in COPY INTO diffs (228 lines): diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c --- a/gdk/gdk_atoms.c +++ b/gdk/gdk_atoms.c @@ -578,6 +578,24 @@ batWrite(const bat *a, stream *s, size_t return mnstr_writeIntArray(s, (const int *) a, cnt) ? GDK_SUCCEED : GDK_FAIL; } +#ifdef HAVE_HGE +static hge h_pow(hge base, hge exp) { + hge result = 1; +#else +static lng h_pow(lng base, lng exp) { + lng result = 1; +#endif +while (exp) +{ +if (exp & 1) +result *= base; +exp >>= 1; +base *= base; +} +return result; +} + + /* * numFromStr parses the head of the string for a number, accepting an * optional sign. The code has been prepared to continue parsing by @@ -592,9 +610,11 @@ numFromStr(const char *src, int *len, vo int sz = ATOMsize(tp); #ifdef HAVE_HGE hge base = 0; + hge expbase = -1; const hge maxdiv10 = GDK_hge_max / 10; #else lng base = 0; + lng expbase = -1; const lng maxdiv10 = LL_CONSTANT(922337203685477580); /*7*/ #endif const int maxmod10 = 7; /* max value % 10 */ @@ -636,8 +656,22 @@ numFromStr(const char *src, int *len, vo } base = 10 * base + base10(*p); p++; + if (*p == 'E' || *p == 'e') { + expbase = base; + base = 0; + p++; + } } while (num10(*p)); + if (expbase > -1) { + dbl checkval = fabs(expbase * pow(10, base)); + if (checkval >= maxdiv10 * 10) { + memcpy(*dst, ATOMnilptr(tp), sz); + return 0; + } + base = expbase * h_pow(10, base); + } base *= sign; + switch (sz) { case 1: { bte **dstbte = (bte **) dst; diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All --- a/sql/backends/monet5/Tests/All +++ b/sql/backends/monet5/Tests/All @@ -59,3 +59,5 @@ HAVE_LIBPY?pyapi29 simpleSQL limithack + +HAVE_HUGEINT?int_notation_1e5 diff --git a/sql/backends/monet5/Tests/int_notation_1e5.sql b/sql/backends/monet5/Tests/int_notation_1e5.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/int_notation_1e5.sql @@ -0,0 +1,13 @@ +START TRANSACTION; +CREATE TABLE t1e4 (a HUGEINT, b HUGEINT); +COPY 7 RECORDS INTO t1e4 FROM STDIN NULL AS ''; +01E4|1| +1E4|1| +1E5|10| +-2E5|-20| +-002E015|-2000| +1E38|100| +-42E36|-42| + +SELECT *, a-b AS diff FROM t1e4; +ROLLBACK; diff --git a/sql/backends/monet5/Tests/int_notation_1e5.stable.err b/sql/backends/monet5/Tests/int_notation_1e5.stable.err new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/int_notation_1e5.stable.err @@ -0,0 +1,36 @@ +stderr of test 'int_notation_1e5` in directory 'sql/backends/monet5` itself: + + +# 11:11:20 > +# 11:11:20 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=37652" "--set" "mapi_usock=/var/tmp/mtest-6682/.s.monetdb.37652" "--set" "monet_prompt=" "--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" "--set" "embedded_r=yes" "--set" "embedded_py=true" +# 11:11:20 > + +# builtin opt gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 37652 +# cmdline opt mapi_usock = /var/tmp/mtest-6682/.s.monetdb.37652 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5 +# cmdline opt embedded_r = yes +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 11:11:20 > +# 11:11:20 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-6682" "--port=37652" +# 11:11:20 > + + +# 11:11:20 > +# 11:11:20 > "Done." +# 11:11:20 > + diff --git a/sql/backends/monet5/Tests/int_notation_1e5.stable.out b/sql/backends/monet5/Tests/int_notation_1e5.stable.out new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/Tests/int_notation_1e5.stable.out @@
MonetDB: pythonloader - second stage COPY INTO sometable FROM LO...
Changeset: 90f85fc4830a for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90f85fc4830a Added Files: monetdb5/extras/pyapi/emit.c monetdb5/extras/pyapi/emit.h monetdb5/extras/pyapi/pyloader.c sql/backends/monet5/Tests/pyapi30.stable.err sql/backends/monet5/Tests/pyapi30.stable.out sql/backends/monet5/Tests/pyapi31.sql sql/backends/monet5/Tests/pyapi31.stable.err sql/backends/monet5/Tests/pyapi31.stable.out Modified Files: monetdb5/extras/pyapi/Makefile.ag monetdb5/extras/pyapi/formatinput.c monetdb5/extras/pyapi/formatinput.h monetdb5/extras/pyapi/pyapi.c monetdb5/extras/pyapi/pyapi.h monetdb5/extras/pyapi/pyapi.mal monetdb5/extras/rapi/rapi.c sql/backends/monet5/Tests/All sql/backends/monet5/Tests/pyapi30.sql sql/backends/monet5/sql_gencode.c sql/include/sql_catalog.h sql/server/rel_psm.c sql/server/rel_select.c sql/server/rel_updates.c sql/server/sql_mvc.h sql/server/sql_parser.y Branch: pythonloader Log Message: second stage COPY INTO sometable FROM LOADER somepythonfunction(); basic functionality is there diffs (truncated from 1512 to 300 lines): diff --git a/monetdb5/extras/pyapi/Makefile.ag b/monetdb5/extras/pyapi/Makefile.ag --- a/monetdb5/extras/pyapi/Makefile.ag +++ b/monetdb5/extras/pyapi/Makefile.ag @@ -24,7 +24,7 @@ MTSAFE lib__pyapi = { MODULE DIR = libdir/monetdb5 - SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c connection.h unspecified_evil.h + SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c connection.h unspecified_evil.h pyloader.c emit.h emit.c XDEPS = $(libpy_LIBDEP) LIBS = ../../tools/libmonetdb5 \ ../../../gdk/libbat \ diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c new file mode 100644 --- /dev/null +++ b/monetdb5/extras/pyapi/emit.c @@ -0,0 +1,195 @@ + +#include "emit.h" +#include "type_conversion.h" +#include "interprocess.h" + +#if PY_MAJOR_VERSION >= 3 +#define IS_PY3K +#define PyString_CheckExact PyUnicode_CheckExact +#define PyString_FromString PyUnicode_FromString +#endif + +#define scalar_convert(tpe) {\ + tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(&dictEntry, 42, &val); \ + BUNappend(self->cols[i].b, &val, 0); \ + if (msg != MAL_SUCCEED) { \ + PyErr_SetString(PyExc_TypeError, "conversion failed"); /* TODO: better error message */ \ + return NULL; \ + }} + +static PyObject * +_emit_emit(Py_EmitObject *self, PyObject *args) { + size_t i, ai; + ssize_t el_count = -1; + str msg = MAL_SUCCEED; + (void) self; + if (!PyDict_Check(args)) { + // complain + PyErr_SetString(PyExc_TypeError, "need dict"); + return NULL; + } + + for (i = 0; i < self->ncols; i++) { + PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name); + ssize_t this_size = 1; + if (dictEntry) { + if (!PyType_IsPyScalar(dictEntry)) { + this_size = Py_SIZE(dictEntry); + } + if (el_count < 0) el_count = this_size; + else { + if (el_count != this_size) { + PyErr_SetString(PyExc_TypeError, "need same length values"); + // todo: better error message! + return NULL; + } + } + } + } + if (el_count < 1) { + PyErr_SetString(PyExc_TypeError, "need at least some values"); + // todo: better error message! + return NULL; + } + + // TODO: check for dict entries not matched by any column and complain if present + + + for (i = 0; i < self->ncols; i++) { + PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name); + if (dictEntry) { + if (PyType_IsPyScalar(dictEntry)) { + switch (self->cols[i].b->T->type) + { + case TYPE_bit: + scalar_convert(bit); + break; + case TYPE_bte: + scalar_convert(bte); + break; + case TYPE_sht: + scalar_convert(sht);
MonetDB: pythonloader - directly creating output tables for crea...
Changeset: 751e80b51d1c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=751e80b51d1c Added Files: sql/backends/monet5/Tests/pyloader07.sql sql/backends/monet5/Tests/pyloader07.stable.err sql/backends/monet5/Tests/pyloader07.stable.out Modified Files: monetdb5/extras/pyapi/connection.c sql/backends/monet5/Tests/All sql/backends/monet5/sql.c sql/backends/monet5/sql.h Branch: pythonloader Log Message: directly creating output tables for create table from loader diffs (truncated from 373 to 300 lines): diff --git a/monetdb5/extras/pyapi/connection.c b/monetdb5/extras/pyapi/connection.c --- a/monetdb5/extras/pyapi/connection.c +++ b/monetdb5/extras/pyapi/connection.c @@ -244,106 +244,76 @@ str _connection_query(Client cntxt, char return res; } -static str _connection_append_table(Client cntxt, char *sname, char *tname, EmitCol *columns, size_t ncols) { + +str _connection_create_table(Client cntxt, char *sname, char *tname, EmitCol *columns, size_t ncols) { size_t i; -size_t nvar = 6; // variables we need to make up -MalBlkRecord mb; -MalStack* stk = NULL; -InstrRecord* pci = NULL; -str res = MAL_SUCCEED; -VarRecord bat_varrec; -mvc* m = ((backend *) cntxt->sqlcontext)->mvc; +sql_table *t; +sql_schema *s; +mvc *sql = NULL; +str msg = MAL_SUCCEED; -assert(tname != NULL && columns != NULL && ncols > 0); + if ((msg = getSQLContext(cntxt, NULL, &sql, NULL)) != NULL) + return msg; + if ((msg = checkSQLContext(cntxt)) != NULL) + return msg; -// very black MAL magic below -mb.var = GDKmalloc(nvar * sizeof(VarRecord*)); -stk = GDKmalloc(sizeof(MalStack) + nvar * sizeof(ValRecord)); -pci = GDKmalloc(sizeof(InstrRecord) + nvar * sizeof(int)); -assert(mb.var != NULL && stk != NULL && pci != NULL); // cough, cough -bat_varrec.type = TYPE_bat; -for (i = 0; i < nvar; i++) { -pci->argv[i] = i; -} -stk->stk[0].vtype = TYPE_int; -stk->stk[2].val.sval = (str) sname; -stk->stk[2].vtype = TYPE_str; -stk->stk[3].val.sval = (str) tname; -stk->stk[3].vtype = TYPE_str; -stk->stk[4].vtype = TYPE_str; -stk->stk[5].vtype = TYPE_bat; -mb.var[5] = &bat_varrec; -for (i = 0; i < ncols; i++) { -EmitCol col = columns[i]; -stk->stk[4].val.sval = col.name; -stk->stk[5].val.bval = col.b->batCacheid; + /* for some reason we don't have an allocator here so make one */ + sql->sa = sa_create(); -res = (*mvc_append_wrap_ptr)(cntxt, &mb, stk, pci); -if (res != NULL) { -break; +if (!sname) sname = "sys"; + if (!(s = mvc_bind_schema(sql, sname))) { + msg = sql_error(sql, 02, "3F000!CREATE TABLE: no such schema '%s'", sname); + goto cleanup; + } + if (!(t = mvc_create_table(sql, s, tname, tt_table, 0, SQL_DECLARED_TABLE, CA_COMMIT, -1))) { + msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not create table '%s'", tname); + goto cleanup; + } + +for(i = 0; i < ncols; i++) { +BAT *b = columns[i].b; +sql_subtype *tpe = sql_bind_localtype(ATOMname(b->T->type)); +sql_column *col = NULL; + +if (!tpe) { + msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not find type for column"); + goto cleanup; +} + +col = mvc_create_column(sql, t, columns[i].name, tpe); +if (!col) { + msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not create column %s", columns[i].name); + goto cleanup; } } -if (res == MAL_SUCCEED) { -(*sqlcleanup_ptr)(m, 0); +msg = create_table_or_view(sql, sname, t, 0); +if (msg != MAL_SUCCEED) { + goto cleanup; } -GDKfree(mb.var); -GDKfree(stk); -GDKfree(pci); -return res; -} +t = mvc_bind_table(sql, s, tname); +if (!t) { + msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not bind table %s", tname); + goto cleanup; +} +for(i = 0; i < ncols; i++) { +BAT *b = columns[i].b; +sql_column *col = NULL; -static char *BatType_ToSQLType(int type) { -switch (type) { -case TYPE_bit: -case TYPE_bte: return "TINYINT"; -case TYPE_sht: return "SMALLINT"; -case TYPE_int: return "INTEGER"; -case TYPE_lng: return "BIGINT"; -case TYPE_flt: return "FLOAT"; -case TYPE_dbl: return "DOUBLE"; -case TYPE_str: return "STRING"; -case TYPE_hge: return "HUGEINT"; -case TYPE_oid: return "UNKNOWN"; -default: return "UNKNOWN"; -} -} - -str _connection_create_table(Client cntxt, char *sname, char *tname, EmitCol *columns, size_t ncols) { -char *query = NULL; -size_t i; -size_t query_size = 255, query_index = 0; -r
MonetDB: default - method to set a memory limit after which mall...
Changeset: 1f5a6fd13a28 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1f5a6fd13a28 Modified Files: gdk/gdk_utils.c gdk/gdk_utils.h monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/mal_io.h monetdb5/modules/mal/mal_io.mal Branch: default Log Message: method to set a memory limit after which malloc fails for testing purposes diffs (97 lines): diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c --- a/gdk/gdk_utils.c +++ b/gdk/gdk_utils.c @@ -310,6 +310,7 @@ int GDK_vm_trim = 1; * fall-back for other compilers. */ #include "gdk_atomic.h" static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0; +static volatile ATOMIC_TYPE GDK_mallocedbytes_limit = 0; static volatile ATOMIC_TYPE GDK_vm_cursize = 0; #ifdef GDK_VM_KEEPHISTO volatile ATOMIC_TYPE GDK_vm_nallocs[MAX_BIT] = { 0 }; @@ -1681,6 +1682,11 @@ GDKmalloc_prefixsize(size_t size) return s; } +gdk_export void GDKsetmemorylimit(size_t nbytes) { + GDK_mallocedbytes_limit = nbytes; +} + + /* * The emergency flag can be set to force a fatal error if needed. * Otherwise, the caller is able to deal with the lack of memory. @@ -1698,6 +1704,13 @@ GDKmallocmax(size_t size, size_t *maxsiz GDKfatal("GDKmallocmax: called with size " SZFMT "", size); #endif } +#ifndef NDEBUG + /* fail malloc for testing purposes depending on set limit */ + if (GDK_mallocedbytes_limit > 0 && + (GDK_mallocedbytes_estimate + size + MALLOC_EXTRA_SPACE) > GDK_mallocedbytes_limit) { + return NULL; + } +#endif size = (size + 7) & ~7; /* round up to a multiple of eight */ s = GDKmalloc_prefixsize(size); if (s == NULL) { @@ -1710,6 +1723,7 @@ GDKmallocmax(size_t size, size_t *maxsiz } GDKfatal("GDKmallocmax: failed for " SZFMT " bytes", size); } else { + /* TODO why are we printing this on stderr? */ fprintf(stderr, "#GDKmallocmax: recovery ok. Continuing..\n"); } } diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h --- a/gdk/gdk_utils.h +++ b/gdk/gdk_utils.h @@ -80,6 +80,9 @@ gdk_export size_t _MT_pagesize; gdk_export void MT_init(void); /* init the package. */ gdk_export int GDKinit(opt *set, int setlen); +/* used for testing only */ +gdk_export void GDKsetmemorylimit(size_t nbytes); + /* * Upon closing the session, all persistent BATs should be saved and * the transient BATs should be removed. The buffer pool manager diff --git a/monetdb5/modules/mal/mal_io.c b/monetdb5/modules/mal/mal_io.c --- a/monetdb5/modules/mal/mal_io.c +++ b/monetdb5/modules/mal/mal_io.c @@ -776,3 +776,11 @@ IOimport(void *ret, bat *bid, str *fnme) return MAL_SUCCEED; } + + +str +IOsetmemorylimit(void *res, lng *nbytes) { + (void) res; + GDKsetmemorylimit(*nbytes); + return MAL_SUCCEED; +} diff --git a/monetdb5/modules/mal/mal_io.h b/monetdb5/modules/mal/mal_io.h --- a/monetdb5/modules/mal/mal_io.h +++ b/monetdb5/modules/mal/mal_io.h @@ -28,4 +28,7 @@ mal_export str IOimport(void *ret, bat * mal_export str io_stdin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); mal_export str io_stdout(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); mal_export str io_stderr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); + +mal_export str IOsetmemorylimit(void *res, lng *nbytes); + #endif /* _PRINT_H_ */ diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal --- a/monetdb5/modules/mal/mal_io.mal +++ b/monetdb5/modules/mal/mal_io.mal @@ -60,3 +60,8 @@ address IOimport comment "Import a BAT from an ASCII dump. The tuples are appended to the first argument. Its signature must match the dump, else parsing errors will occur as an exception."; + +# used for testing malloc failures +command setmemorylimit(nbytes:lng):void +address IOsetmemorylimit +comment "Set memory limit for testing"; \ No newline at end of file ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - fixed crash in mapi code
Changeset: 0c5dff585dd5 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c5dff585dd5 Modified Files: monetdb5/modules/mal/mal_mapi.c Branch: default Log Message: fixed crash in mapi code diffs (15 lines): diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c --- a/monetdb5/modules/mal/mal_mapi.c +++ b/monetdb5/modules/mal/mal_mapi.c @@ -365,6 +365,11 @@ SERVERlistenThread(SOCKET *Sock) fflush(stdout); #endif data = GDKmalloc(sizeof(*data)); + if (!data) { + closesocket(msgsock); + msg = "memory allocation failed"; + goto error; + } data->in = socket_rastream(msgsock, "Server read"); data->out = socket_wastream(msgsock, "Server write"); if (MT_create_thread(&tid, doChallenge, data, MT_THR_JOINABLE)) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - fixed crash on malloc failure
Changeset: 9c5b7d0a3f58 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9c5b7d0a3f58 Modified Files: monetdb5/mal/mal_exception.c monetdb5/mal/mal_instruction.c Branch: default Log Message: fixed crash on malloc failure diffs (39 lines): diff --git a/monetdb5/mal/mal_exception.c b/monetdb5/mal/mal_exception.c --- a/monetdb5/mal/mal_exception.c +++ b/monetdb5/mal/mal_exception.c @@ -141,7 +141,9 @@ showException(stream *out, enum malexcep va_end(ap); dumpExceptionsToStream(out, msg); - GDKfree(msg); + if (msg && msg != M5OutOfMemory) { + GDKfree(msg); + } } /** diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -410,8 +410,10 @@ newInstruction(MalBlkPtr mb, int kind) } if (p == NULL) { p = GDKzalloc(MAXARG * sizeof(p->argv[0]) + offsetof(InstrRecord, argv)); - if (p == NULL) + if (p == NULL) { + showException(GDKout, MAL, "pushEndInstruction", "memory allocation failure"); return NULL; + } p->maxarg = MAXARG; } p->typechk = TYPE_UNKNOWN; @@ -1540,6 +1542,9 @@ pushEndInstruction(MalBlkPtr mb) InstrPtr p; p = newInstruction(mb, ENDsymbol); + if (!p) { + return; + } p->argc = 0; p->retc = 0; p->argv[0] = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - fix compiler warning with clang
Changeset: 9f7fbbf9727c for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9f7fbbf9727c Modified Files: monetdb5/mal/mal_instruction.c Branch: default Log Message: fix compiler warning with clang diffs (12 lines): diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -600,7 +600,7 @@ findVariableLength(MalBlkPtr mb, str nam int j; for (i = mb->vtop - 1; i >= 0; i--) - if (mb->var[i] && mb->var[i]->id) { + if (mb->var[i]) { /* mb->var[i]->id will always evaluate to true */ str s = mb->var[i]->id; j = 0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - first test case using memory allocation limit...
Changeset: 946681f654d8 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=946681f654d8 Added Files: sql/test/malloc_fail/Tests/All sql/test/malloc_fail/Tests/setmemorylimit.sql sql/test/malloc_fail/Tests/setmemorylimit.stable.err sql/test/malloc_fail/Tests/setmemorylimit.stable.out Branch: default Log Message: first test case using memory allocation limit, for now all is well diffs (137 lines): diff --git a/sql/test/malloc_fail/Tests/All b/sql/test/malloc_fail/Tests/All new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/All @@ -0,0 +1,1 @@ +setmemorylimit diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.sql b/sql/test/malloc_fail/Tests/setmemorylimit.sql new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit.sql @@ -0,0 +1,7 @@ +create procedure setmemorylimit(nbytes BIGINT) external name "io"."setmemorylimit"; +call setmemorylimit(0); +call setmemorylimit(1); +-- this should work fine +select name from tables where 1=0; +call setmemorylimit(0); +drop procedure setmemorylimit; diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.stable.err b/sql/test/malloc_fail/Tests/setmemorylimit.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit.stable.err @@ -0,0 +1,34 @@ +stderr of test 'setmemorylimit` in directory 'sql/test/malloc_fail` itself: + + +# 11:39:58 > +# 11:39:58 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36345" "--set" "mapi_usock=/var/tmp/mtest-21164/.s.monetdb.36345" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail" +# 11:39:58 > + +# builtin opt gdk_dbpath = /Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 36345 +# cmdline opt mapi_usock = /var/tmp/mtest-21164/.s.monetdb.36345 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail +# cmdline opt gdk_debug = 536870922 + +# 11:39:58 > +# 11:39:58 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-21164" "--port=36345" +# 11:39:58 > + + +# 11:39:59 > +# 11:39:59 > "Done." +# 11:39:59 > + diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.stable.out b/sql/test/malloc_fail/Tests/setmemorylimit.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit.stable.out @@ -0,0 +1,75 @@ +stdout of test 'setmemorylimit` in directory 'sql/test/malloc_fail` itself: + + +# 11:39:58 > +# 11:39:58 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=36345" "--set" "mapi_usock=/var/tmp/mtest-21164/.s.monetdb.36345" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail" +# 11:39:58 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_sql_test_malloc_fail', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 128bit integers +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36345/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-21164/.s.monetdb.36345 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded + +Ready. +# SQL catalog created, loading sql scripts once +# loading sql script: 09_like.sql +# loading sql script: 10_math.sql +# loading sql script: 11_times.sql +# loading sql script: 12_url.sql +# loading sql script: 13_date.sql +# loading sql script: 14_inet.sql +# loading sql script: 15_querylog.sql +# loading sql script: 16_tracelog.sql +# loading sql script: 17_temporal.sql +# loading sql script: 18_index.sql +# loading sql script: 20_vacuum.sql +# loading sql script: 21_dependency_functions.sql +# loading sql script: 22_clients.sql +# loading sql script: 23_skyserver.sql +# loading sql script: 24_zorder.sql +# loading sql script: 25_debug.sql +# loading sql script: 26_sysmon.sql +# loading sql script: 27_rejects.sql +# loading sql script: 39_analytics.sql +# loading sql script: 39_analytics_hge.sql +# loading sql script: 40_geom.sql +# loading sql script: 40_json.sql +# loading sql script: 40_json_hge.sql +# loading sql script: 41_md5sum.sql +# loading sql script: 45_uuid.sql +# loading s
MonetDB: default - newline
Changeset: d23c1aef84ae for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d23c1aef84ae Modified Files: monetdb5/modules/mal/mal_io.mal Branch: default Log Message: newline diffs (10 lines): diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal --- a/monetdb5/modules/mal/mal_io.mal +++ b/monetdb5/modules/mal/mal_io.mal @@ -64,4 +64,4 @@ comment "Import a BAT from an ASCII dump # used for testing malloc failures command setmemorylimit(nbytes:lng):void address IOsetmemorylimit -comment "Set memory limit for testing"; \ No newline at end of file +comment "Set memory limit for testing"; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - changed semantics of memory limit to only fai...
Changeset: 95fbd5f0b4a0 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=95fbd5f0b4a0 Modified Files: gdk/gdk_utils.c gdk/gdk_utils.h monetdb5/mal/mal_instruction.c monetdb5/modules/mal/mal_mapi.c sql/backends/monet5/sql_scenario.c sql/common/sql_list.c sql/common/sql_mem.c sql/server/rel_semantic.c sql/storage/sql_catalog.c sql/test/malloc_fail/Tests/All sql/test/malloc_fail/Tests/setmemorylimit.sql Branch: default Log Message: changed semantics of memory limit to only fail allocations larger than limit diffs (158 lines): diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c --- a/gdk/gdk_utils.c +++ b/gdk/gdk_utils.c @@ -310,7 +310,7 @@ int GDK_vm_trim = 1; * fall-back for other compilers. */ #include "gdk_atomic.h" static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0; -static volatile ATOMIC_TYPE GDK_mallocedbytes_limit = 0; +static volatile lng GDK_mallocedbytes_limit = -1; static volatile ATOMIC_TYPE GDK_vm_cursize = 0; #ifdef GDK_VM_KEEPHISTO volatile ATOMIC_TYPE GDK_vm_nallocs[MAX_BIT] = { 0 }; @@ -1682,7 +1682,7 @@ GDKmalloc_prefixsize(size_t size) return s; } -gdk_export void GDKsetmemorylimit(size_t nbytes) { +gdk_export void GDKsetmemorylimit(lng nbytes) { GDK_mallocedbytes_limit = nbytes; } @@ -1706,8 +1706,7 @@ GDKmallocmax(size_t size, size_t *maxsiz } #ifndef NDEBUG /* fail malloc for testing purposes depending on set limit */ - if (GDK_mallocedbytes_limit > 0 && - (GDK_mallocedbytes_estimate + size + MALLOC_EXTRA_SPACE) > GDK_mallocedbytes_limit) { + if (GDK_mallocedbytes_limit >= 0 && size >(size_t) GDK_mallocedbytes_limit) { return NULL; } #endif diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h --- a/gdk/gdk_utils.h +++ b/gdk/gdk_utils.h @@ -81,7 +81,7 @@ gdk_export void MT_init(void);/* init gdk_export int GDKinit(opt *set, int setlen); /* used for testing only */ -gdk_export void GDKsetmemorylimit(size_t nbytes); +gdk_export void GDKsetmemorylimit(lng nbytes); /* * Upon closing the session, all persistent BATs should be saved and diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c --- a/monetdb5/mal/mal_instruction.c +++ b/monetdb5/mal/mal_instruction.c @@ -1543,6 +1543,8 @@ pushEndInstruction(MalBlkPtr mb) p = newInstruction(mb, ENDsymbol); if (!p) { + mb->errors++; + showException(GDKout, MAL, "pushEndInstruction", "failed to create instruction (out of memory?)"); return; } p->argc = 0; diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c --- a/monetdb5/modules/mal/mal_mapi.c +++ b/monetdb5/modules/mal/mal_mapi.c @@ -367,8 +367,9 @@ SERVERlistenThread(SOCKET *Sock) data = GDKmalloc(sizeof(*data)); if (!data) { closesocket(msgsock); - msg = "memory allocation failed"; - goto error; + showException(GDKstdout, MAL, "initClient", + "cannot allocate memory"); + continue; } data->in = socket_rastream(msgsock, "Server read"); data->out = socket_wastream(msgsock, "Server write"); diff --git a/sql/backends/monet5/sql_scenario.c b/sql/backends/monet5/sql_scenario.c --- a/sql/backends/monet5/sql_scenario.c +++ b/sql/backends/monet5/sql_scenario.c @@ -936,6 +936,12 @@ SQLparser(Client c) * this point if this is a recursive call. */ if (!m->sa) m->sa = sa_create(); + if (!m->sa) { + mnstr_printf(out, "!Could not create SQL allocator\n"); + mnstr_flush(out); + c->mode = FINISHCLIENT; + throw(SQL, "SQLparser", "Could not create SQL allocator"); + } m->emode = m_normal; m->emod = mod_none; diff --git a/sql/common/sql_list.c b/sql/common/sql_list.c --- a/sql/common/sql_list.c +++ b/sql/common/sql_list.c @@ -24,6 +24,9 @@ list * list_create(fdestroy destroy) { list *l = MNEW(list); + if (!l) { + return NULL; + } l->sa = NULL; l->destroy = destroy; diff --git a/sql/common/sql_mem.c b/sql/common/sql_mem.c --- a/sql/common/sql_mem.c +++ b/sql/common/sql_mem.c @@ -44,7 +44,9 @@ sql_ref_dec(sql_ref *r) sql_allocator *sa_create(void) { sql_allocator *sa = MNEW(sql_allocator); - + if (!sa) { + return NULL; + } sa->size = 64; sa->nr = 1; sa->blks = NEW_ARRAY(char*,sa->size); diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c --- a/sql/server/rel_semantic.c +++ b/sql/server/rel_semantic.c @@ -43,6 +43,9 @@ rel_parse(mvc *m, sql_schema *s, char *q b = (buf
MonetDB: default - new tests
Changeset: dc8dad5e60d3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dc8dad5e60d3 Added Files: sql/test/malloc_fail/Tests/setmemorylimit-fail.sql sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql Branch: default Log Message: new tests diffs (98 lines): diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql b/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql @@ -0,0 +1,3 @@ +create procedure setmemorylimit(nbytes BIGINT) external name "io"."setmemorylimit"; +call setmemorylimit(1000); +select name from tables where 1=0; diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err @@ -0,0 +1,40 @@ +stderr of test 'setmemorylimit-fail` in directory 'sql/test/malloc_fail` itself: + + +# 15:41:50 > +# 15:41:50 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=38119" "--set" "mapi_usock=/var/tmp/mtest-56431/.s.monetdb.38119" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail" +# 15:41:50 > + +# builtin opt gdk_dbpath = /Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 38119 +# cmdline opt mapi_usock = /var/tmp/mtest-56431/.s.monetdb.38119 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail +# cmdline opt gdk_debug = 536870922 + +# 15:41:50 > +# 15:41:50 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-56431" "--port=38119" +# 15:41:50 > + +MAPI = (monetdb) /var/tmp/mtest-56431/.s.monetdb.38119 +QUERY = create procedure setmemorylimit(nbytes BIGINT) external name "io"."setmemorylimit"; +ERROR = !CREATE PROCEDURE: name 'setmemorylimit' (bigint(64)) already in use +MAPI = (monetdb) /var/tmp/mtest-56431/.s.monetdb.38119 +QUERY = select name from tables where 1=0; +ERROR = !Could not create SQL allocator + +# 15:41:50 > +# 15:41:50 > "Done." +# 15:41:50 > + diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out @@ -0,0 +1,32 @@ +stdout of test 'setmemorylimit-fail` in directory 'sql/test/malloc_fail` itself: + + +# 15:41:50 > +# 15:41:50 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=38119" "--set" "mapi_usock=/var/tmp/mtest-56431/.s.monetdb.38119" "--set" "monet_prompt=" "--forcemito" "--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail" +# 15:41:50 > + +# MonetDB 5 server v11.24.0 +# This is an unreleased version +# Serving database 'mTests_sql_test_malloc_fail', using 4 threads +# Compiled for x86_64-apple-darwin15.5.0/64bit with 128bit integers +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:38119/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-56431/.s.monetdb.38119 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded + +Ready. +!Memory allocation failed. + +# 15:41:50 > +# 15:41:50 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-56431" "--port=38119" +# 15:41:50 > + + +# 15:41:50 > +# 15:41:50 > "Done." +# 15:41:50 > + diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql b/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql new file mode 100644 --- /dev/null +++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql @@ -0,0 +1,3 @@ +create procedure setmemorylimit(nbytes BIGINT) external name "io"."setmemorylimit"; +call setmemorylimit(0); +select name from tables where 1=0; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pythonloader - xEy integer handling improved
Changeset: ddfd4a5e4829 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ddfd4a5e4829 Modified Files: gdk/gdk_atoms.c Branch: pythonloader Log Message: xEy integer handling improved diffs (67 lines): diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c --- a/gdk/gdk_atoms.c +++ b/gdk/gdk_atoms.c @@ -578,23 +578,6 @@ batWrite(const bat *a, stream *s, size_t return mnstr_writeIntArray(s, (const int *) a, cnt) ? GDK_SUCCEED : GDK_FAIL; } -#ifdef HAVE_HGE -static hge h_pow(hge base, hge exp) { - hge result = 1; -#else -static lng h_pow(lng base, lng exp) { - lng result = 1; -#endif -while (exp) -{ -if (exp & 1) -result *= base; -exp >>= 1; -base *= base; -} -return result; -} - /* * numFromStr parses the head of the string for a number, accepting an @@ -656,19 +639,34 @@ numFromStr(const char *src, int *len, vo } base = 10 * base + base10(*p); p++; + /* Special case: xEy = x*10^y handling part 1 */ if (*p == 'E' || *p == 'e') { + // if there is a second E in the string we give up + if (expbase > -1) { + memcpy(*dst, ATOMnilptr(tp), sz); + return 0; + } expbase = base; base = 0; p++; } } while (num10(*p)); + /* Special case: xEy = x*10^y handling part 2 */ if (expbase > -1) { - dbl checkval = fabs(((dbl) expbase) * (dbl) pow(10, (dbl) base)); - if (checkval >= maxdiv10 * 10) { - memcpy(*dst, ATOMnilptr(tp), sz); - return 0; +#ifdef HAVE_HGE + hge res = expbase; +#else + lng res = expbase; +#endif + while (base > 0) { + if (res > maxdiv10) { + memcpy(*dst, ATOMnilptr(tp), sz); + return 0; + } + res *= 10L; + base--; } - base = expbase * h_pow(10, base); + base = res; } base *= sign; ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: protocol - new branch for protocol rewrite
Changeset: eb1b9781c58f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=eb1b9781c58f Modified Files: clients/mapilib/mapi.c common/stream/stream.c monetdb5/modules/mal/mal_mapi.c sql/backends/monet5/sql_result.c Branch: protocol Log Message: new branch for protocol rewrite diffs (79 lines): diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -2535,6 +2535,8 @@ mapi_reconnect(Mapi mid) mid->connected = 1; + // FIXME: these need to become snappy streams (framing2) + if (!isa_block_stream(mid->to)) { mid->to = block_stream(mid->to); check_stream(mid, mid->to, mnstr_error(mid->to), "mapi_reconnect", mid->error); @@ -3930,6 +3932,7 @@ read_into_cache(MapiHdl hdl, int lookahe break; case '%': case '#': + //FIXME: add new result set parsing case '&': if (lookahead < 0) lookahead = 1; diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -3921,6 +3921,11 @@ bs_clrerr(stream *s) mnstr_clearerr(((bs *) s->stream_data.p)->s); } + +// FIXME: patch bs_read/bs_write etc +// 10 MB max buffer size or so +// unsigned int length or so + stream * block_stream(stream *s) { diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c --- a/monetdb5/modules/mal/mal_mapi.c +++ b/monetdb5/modules/mal/mal_mapi.c @@ -114,6 +114,8 @@ doChallenge(void *data) char *buf = (char *) GDKmalloc(BLOCK + 1); char challenge[13]; char *algos; + + // FIXME: make this a snappy stream as well stream *fdin = block_stream(((struct challengedata *) data)->in); stream *fdout = block_stream(((struct challengedata *) data)->out); bstream *bs; @@ -142,8 +144,10 @@ doChallenge(void *data) /* generate the challenge string */ generateChallenge(challenge, 8, 12); algos = mcrypt_getHashAlgorithms(); + // FIXME: add the newproto flag to algos and rename to 'capabilities' to hide the crime + /* note that we claim to speak proto 9 here for hashed passwords */ - mnstr_printf(fdout, "%s:mserver:9:%s:%s:%s:", + mnstr_printf(fdout, "%s:mserver:10:%s:%s:%s:", challenge, algos, #ifdef WORDS_BIGENDIAN @@ -171,6 +175,7 @@ doChallenge(void *data) mnstr_printf(cntxt->fdout, "#SERVERlisten:client accepted\n"); mnstr_printf(cntxt->fdout, "#SERVERlisten:client string %s\n", buf); #endif + // FIXME: how can we avoid this bs = bstream_create(fdin, 128 * BLOCK); if (bs == NULL){ diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c --- a/sql/backends/monet5/sql_result.c +++ b/sql/backends/monet5/sql_result.c @@ -1313,6 +1313,8 @@ mvc_export_row(backend *b, stream *s, re return (ok) ? 0 : -1; } + +//FIXME: rewrite this into something new static int mvc_export_table(backend *b, stream *s, res_table *t, BAT *order, BUN offset, BUN nr, char *btag, char *sep, char *rsep, char *ssep, char *ns) { ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: protocol - Additional changes to byte_stream.
Changeset: b60c634a07cd for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b60c634a07cd Modified Files: clients/mapilib/mapi.c common/stream/stream.c common/stream/stream.h Branch: protocol Log Message: Additional changes to byte_stream. diffs (truncated from 483 to 300 lines): diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -2781,15 +2781,13 @@ mapi_reconnect(Mapi mid) check_stream(mid, mid->to, "Could not send initial byte sequence", "mapi_reconnect", mid->error); if (prot_version == prot10 || prot_version == prot10compressed) { + bstream *bs_to = (bstream*) mid->to; + bstream *bs_from = (bstream*) mid->from; + printf("Using protocol version %s.\n", prot_version == prot10 ? "PROT10" : "PROT10COMPRESSED"); - // FIXME: destroy block streams and replace with appropriate streams -#if 0 assert(isa_block_stream(mid->to)); assert(isa_block_stream(mid->from)); - bs *bs_to = (bs*) mid->to; - bs *bs_from = (bs*) mid->from; - if (prot_version == prot10compressed) { #ifdef HAVE_LIBSNAPPY mid->to = compressed_stream(bs_to->s, COMPRESSION_SNAPPY); @@ -2798,13 +2796,16 @@ mapi_reconnect(Mapi mid) assert(0); #endif } else { - mid->to = byte_stream((bs_to->s); - mid->from = byte_stream(bs_from->s); + // FIXME: figure out proper stream sizes + mid->to = byte_stream(bs_to->s, 1024000); + mid->from = byte_stream(bs_from->s, 1024000); + } - - close_stream(bs_to); - close_stream(bs_from); -#endif + bs_to->s = NULL; + bs_from->s = NULL; + close_stream((stream*) bs_to); + close_stream((stream*) bs_from); + } /* consume the welcome message from the server */ diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -3885,20 +3885,6 @@ bs_isalive(stream *ss) } static void -bs_close(stream *ss) -{ - bs *s; - - s = (bs *) ss->stream_data.p; - assert(s); - if (s == NULL) - return; - assert(s->s); - if (s->s) - s->s->close(s->s); -} - -static void bs_destroy(stream *ss) { bs *s; @@ -3914,6 +3900,24 @@ bs_destroy(stream *ss) destroy(ss); } + + +static void +bs_close(stream *ss) +{ + bs *s; + + s = (bs *) ss->stream_data.p; + assert(s); + if (s == NULL) + return; + assert(s->s); + if (s->s) + s->s->close(s->s); + bs_destroy(ss); +} + + static void bs_clrerr(stream *s) { @@ -3984,8 +3988,9 @@ bytestream_create(stream *s, size_t bufs return NULL; ns->s = s; ns->bufsize = bufsize; - ns->bufpos = 0; + ns->bufpos = BYTESTREAM_OVERHEAD; ns->buf = malloc(bufsize); + ns->bufend = ns->bufpos; if (ns->buf == NULL) { free(ns); return NULL; @@ -4003,20 +4008,17 @@ bytestream_flush(stream *ss) return -1; assert(ss->access == ST_WRITE); assert(s->bufpos < s->bufsize); - if (ss->access == ST_WRITE) { - if (s->bufpos + BYTESTREAM_OVERHEAD <= s->bufsize) { - // if there is room in the buffer, end the buffer with a 0 so the reader knows that the stream has ended - lng data = 0; - memcpy(s->buf + s->bufpos, &data, sizeof(lng)); - s->bufpos += sizeof(lng); - } - // FIXME: for compressed stream we can compress the buffer here before writing it - if (!s->s->write(s->s, s->buf, 1, s->bufpos)) { - ss->errnr = MNSTR_WRITE_ERROR; - return -1; - } - s->bufpos = 0; + + memcpy(s->buf, &s->bufpos, sizeof(size_t)); + + + // FIXME: for compressed stream we can compress the buffer here before writing it + if (!s->s->write(s->s, s->buf, 1, s->bufpos)) { + ss->errnr = MNSTR_WRITE_ERROR; + return -1; } + s->bufpos = BYTESTREAM_OVERHEAD; + return 0; } @@ -4031,227 +4033,125 @@ bytestream_write(stream *ss, const void return -1; assert(ss->access == ST_WRITE); assert(s->bufpos < s->bufsize); - if (todo + BYTESTREAM_OVERHEAD > s->bufsize) + if (todo > s->bufsize) fprintf(stderr, "Content too big for buffer!\n"); ss->errnr = MNSTR_WRITE_ERROR; return -1; // content does not fit into
MonetDB: protocol - mapi version 10, bigger and snappier blocks
Changeset: a2e6bf7213e6 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a2e6bf7213e6 Modified Files: clients/mapilib/mapi.c common/stream/Makefile.ag common/stream/stream.c common/stream/stream.h configure.ag monetdb5/modules/mal/mal_mapi.c Branch: protocol Log Message: mapi version 10, bigger and snappier blocks diffs (truncated from 894 to 300 lines): diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -2214,6 +2214,8 @@ mapi_reconnect(Mapi mid) char *protover; char *rest; protocol_version prot_version = prot9; + // FIXME: make this configurable + size_t block_size = 1024000; if (mid->connected) close_connection(mid); @@ -2732,7 +2734,7 @@ mapi_reconnect(Mapi mid) if (prot_version == prot10 || prot_version == prot10compressed) { // if we are using protocol 10, we have to send either PROT10/PROT10COMPRESSED to the server // so the server knows which protocol to use - retval = snprintf(buf, BLOCK, "%s:%s:%s:%s:%s:%s:\n", + retval = snprintf(buf, BLOCK, "%s:%s:%s:%s:%s:%s:%zu:\n", #ifdef WORDS_BIGENDIAN "BIG", #else @@ -2740,7 +2742,8 @@ mapi_reconnect(Mapi mid) #endif mid->username, hash, mid->language, mid->database == NULL ? "" : mid->database, -prot_version == prot10 ? "PROT10" : "PROT10COMPRESSED"); +prot_version == prot10 ? "PROT10" : "PROT10COMPRESSED", + block_size); } else { retval = snprintf(buf, BLOCK, "%s:%s:%s:%s:%s:\n", #ifdef WORDS_BIGENDIAN @@ -2781,15 +2784,13 @@ mapi_reconnect(Mapi mid) check_stream(mid, mid->to, "Could not send initial byte sequence", "mapi_reconnect", mid->error); if (prot_version == prot10 || prot_version == prot10compressed) { - bstream *bs_to = (bstream*) mid->to; - bstream *bs_from = (bstream*) mid->from; printf("Using protocol version %s.\n", prot_version == prot10 ? "PROT10" : "PROT10COMPRESSED"); assert(isa_block_stream(mid->to)); assert(isa_block_stream(mid->from)); if (prot_version == prot10compressed) { -#ifdef HAVE_LIBSNAPPY +#ifdef HAVE_LIBSNAPPY2 mid->to = compressed_stream(bs_to->s, COMPRESSION_SNAPPY); mid->from = compressed_stream(bs_from->s, COMPRESSION_SNAPPY); #else @@ -2797,14 +2798,15 @@ mapi_reconnect(Mapi mid) #endif } else { // FIXME: figure out proper stream sizes - mid->to = byte_stream(bs_to->s, 1024000); - mid->from = byte_stream(bs_from->s, 1024000); - + mid->to = block_stream2(bs_stream(mid->to), block_size); + mid->from = block_stream2(bs_stream(mid->from), block_size); } - bs_to->s = NULL; - bs_from->s = NULL; - close_stream((stream*) bs_to); - close_stream((stream*) bs_from); + + // FIXME: this leaks +// bs_to->s = NULL; +// bs_from->s = NULL; +// close_stream((stream*) bs_to); +// close_stream((stream*) bs_from); } diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag --- a/common/stream/Makefile.ag +++ b/common/stream/Makefile.ag @@ -10,6 +10,7 @@ MTSAFE INCLUDES = $(zlib_CFLAGS) \ $(BZ_CFLAGS) \ + $(snappy_CFLAGS) \ $(liblzma_CFLAGS) \ $(openssl_CFLAGS) \ $(curl_CFLAGS) @@ -20,6 +21,7 @@ lib_stream = { LIBS = $(SOCKET_LIBS) \ $(zlib_LIBS) \ $(BZ_LIBS) \ + $(snappy_LIBS) \ $(liblzma_LIBS) \ $(openssl_LIBS) \ $(curl_LIBS) \ diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -3567,6 +3567,8 @@ buffer_wastream(buffer *b, const char *n return s; } + + /* -- */ /* A buffered stream consists of a sequence of blocks. Each block @@ -3885,6 +3887,20 @@ bs_isalive(stream *ss) } static void +bs_close(stream *ss) +{ + bs *s; + + s = (bs *) ss->stream_data.p; + assert(s); + if (s == NULL) + return; + assert(s->s); + if (s->s) +
MonetDB: protocol - mclient support for new block stream, compre...
Changeset: 721c8b3d945f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=721c8b3d945f Modified Files: clients/mapiclient/mclient.c clients/mapilib/mapi.c clients/mapilib/mapi.h common/stream/stream.c common/stream/stream.h monetdb5/modules/mal/mal_mapi.c Branch: protocol Log Message: mclient support for new block stream, compression working diffs (truncated from 615 to 300 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -2932,6 +2932,10 @@ usage(const char *prog, int xit) fprintf(stderr, " -E charset | --encoding=charset specify encoding (character set) of the terminal\n"); #endif fprintf(stderr, " -f kind | --format=kind specify output format {csv,tab,raw,sql,xml}\n"); + + fprintf(stderr, " -P version | --protocol=version specify protocol version {prot9,prot10,prot10compressed}\n"); + fprintf(stderr, " -B size | --blocksize=size specify protocol block size (>= %d)\n", BLOCK); + fprintf(stderr, " -H | --history load/save cmdline history (default off)\n"); fprintf(stderr, " -i | --interactive[=tm] interpret `\\' commands on stdin, use time formatting {ms,s,m}\n"); fprintf(stderr, " -l language | --language=lang{sql,mal}\n"); @@ -2969,6 +2973,8 @@ main(int argc, char **argv) char *command = NULL; char *dbname = NULL; char *output = NULL;/* output format as string */ + char *protocol = NULL; + size_t blocksize = 0; FILE *fp = NULL; int trace = 0; int dump = 0; @@ -2992,6 +2998,9 @@ main(int argc, char **argv) {"encoding", 1, 0, 'E'}, #endif {"format", 1, 0, 'f'}, + {"protocol", 1, 0, 'P'}, + {"blocksize", 1, 0, 'B'}, + {"help", 0, 0, '?'}, {"history", 0, 0, 'H'}, {"host", 1, 0, 'h'}, @@ -3125,6 +3134,16 @@ main(int argc, char **argv) free(output); output = strdup(optarg);/* output format */ break; + case 'P': + assert(optarg); + if (protocol != NULL) + free(protocol); + protocol = strdup(optarg); + break; + case 'B': + assert(optarg); + blocksize = (size_t) atol(optarg); + break; case 'i': interactive = 1; showtiming = 1; @@ -3271,6 +3290,30 @@ main(int argc, char **argv) if (passwd) free(passwd); passwd = NULL; + + if (blocksize > 0) { + if (blocksize < BLOCK) { + fprintf(stderr, "invalid block size (needs to be bigger than %d)\n", BLOCK); + } else { + mapi_set_blocksize(mid, blocksize); + } + } + + if (protocol) { + if (strcasecmp(protocol, "prot9") == 0) { + mapi_set_protocol(mid, prot9); + } + else if (strcasecmp(protocol, "prot10") == 0) { + mapi_set_protocol(mid, prot10); + } + else if (strcasecmp(protocol, "prot10compressed") == 0) { + mapi_set_protocol(mid, prot10compressed); + } + else { + fprintf(stderr, "invalid protocol name '%s'\n", protocol); + } + } + if (mid && mapi_error(mid) == MOK) mapi_reconnect(mid);/* actually, initial connect */ diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -881,6 +881,7 @@ struct BlockCache { int eos;/* end of sequence */ }; + /* A connection to a server is represented by a struct MapiStruct. An application can have any number of connections to any number of servers. Connections are completely independent of each other. @@ -897,6 +898,8 @@ struct MapiStruct { char *uri; int languageId; char *motd; /* welcome message from server */ + protocol_version protocol; + size_t blocksize; int trace; /* Trace Mapi interaction */ int auto_commit; @@ -1884,6 +1887,9 @@ mapi_new(void) mid->username = NULL; mid->password = NULL; + mid->protocol = protauto; + mid->blocksize = 128*BLOCK; // 1 MB + mid->cachelimit = 100; mid->redircnt = 0; mid->redirmax = 10; @@ -2193,11 +2199,6 @@ mapi_destroy(Mapi mid) return MOK; } -typedef enum { - prot9 = 1, -
MonetDB: protocol - merge with default
Changeset: edcb9d8a2c8f for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=edcb9d8a2c8f Removed Files: monetdb5/modules/mal/Tests/zorder.malC monetdb5/modules/mal/Tests/zorder.stable.err monetdb5/modules/mal/Tests/zorder.stable.out monetdb5/modules/mal/zorder.c monetdb5/modules/mal/zorder.h monetdb5/modules/mal/zorder.mal sql/scripts/24_zorder.sql Modified Files: NT/installer32/MonetDB5-SQL-Installer.vdproj NT/installer64/MonetDB5-SQL-Installer.vdproj clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/SQL-dump.stable.out clients/Tests/SQL-dump.stable.out.int128 clients/Tests/exports.stable.out clients/odbc/driver/README clients/odbc/driver/SQLBindParameter.c clients/odbc/driver/SQLBulkOperations.c clients/odbc/driver/SQLCopyDesc.c clients/odbc/driver/SQLDescribeParam.c clients/odbc/driver/SQLEndTran.c clients/odbc/driver/SQLExtendedFetch.c clients/odbc/driver/SQLFetchScroll.c clients/odbc/driver/SQLGetEnvAttr.c clients/odbc/driver/SQLNumParams.c clients/odbc/driver/SQLSetDescRec.c clients/odbc/driver/SQLSetEnvAttr.c configure.ag java/src/main/java/nl/cwi/monetdb/jdbc/MonetBlob.java java/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in monetdb5/ChangeLog monetdb5/mal/mal.h monetdb5/mal/mal_instruction.c monetdb5/mal/mal_instruction.h monetdb5/mal/mal_listing.c monetdb5/mal/mal_module.c monetdb5/mal/mal_module.h monetdb5/mal/mal_parser.c monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/Tests/All monetdb5/modules/mal/mal_init.mal monetdb5/optimizer/opt_profiler.c rpm.mk.in sql/ChangeLog sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_upgrades.c sql/scripts/Makefile.ag sql/test/BugTracker-2016/Tests/All sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit sql/test/Tests/systemfunctions.stable.out sql/test/Tests/systemfunctions.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/check.stable.out sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128 sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128 sql/test/emptydb-upgrade/Tests/check.stable.out sql/test/emptydb-upgrade/Tests/check.stable.out.int128 sql/test/emptydb-upgrade/Tests/upgrade.stable.out sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128 sql/test/emptydb/Tests/check.stable.out sql/test/emptydb/Tests/check.stable.out.int128 sql/test/leaks/Tests/check1.stable.out.int128 sql/test/leaks/Tests/check2.stable.out.int128 sql/test/leaks/Tests/check3.stable.out.int128 sql/test/leaks/Tests/check4.stable.out.int128 sql/test/leaks/Tests/check5.stable.out.int128 sql/test/pg_regress/Tests/strings.stable.err sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128 sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out sql/test/testdb-upgrade/Tests/upgrade.stable.out sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128 Branch: protocol Log Message: merge with default diffs (truncated from 5860 to 300 lines): diff --git a/NT/installer32/MonetDB5-SQL-Installer.vdproj b/NT/installer32/MonetDB5-SQL-Installer.vdproj --- a/NT/installer32/MonetDB5-SQL-Installer.vdproj +++ b/NT/installer32/MonetDB5-SQL-Installer.vdproj @@ -897,12 +897,6 @@ } "Entry" { -"MsmKey" = "8:_EC56B031A65A44BCA049E43F5A7641C4" -"OwnerKey" = "8:_UNDEFINED" -"MsmSig" = "8:_UNDEFINED" -} -"Entry" -{ "MsmKey" = "8:_F0AD495DD01A481B8FDB22499521AFB7" "OwnerKey" = "8:_UNDEFINED" "MsmSig" = "8:_UNDEFINED" @@ -939,12 +933,6 @@ } "Entry" { -"MsmKey" = "8:_F64E69D474DA4F6A89A1B014F77E4669" -"OwnerKey" = "8:_UNDEFINED" -"MsmSig" = "8:_UNDEFINED" -} -"Entry" -{ "MsmKey" = "8:_F827A21B394243CA9A2DA27D42DA0C93" "OwnerKey" = "8:_UNDEFINED" "MsmSig" = "8:_UNDEFINED" @@ -3964,26 +3952,6 @@ "Is
MonetDB: protocol - compressed streams actually working for larg...
Changeset: 7a7bd9024d74 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7a7bd9024d74 Modified Files: clients/mapiclient/mclient.c clients/mapilib/mapi.c clients/mapilib/mapi.h common/stream/stream.c common/stream/stream.h monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_function.c monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/modules/mal/mal_mapi.c sql/backends/monet5/sql.c sql/backends/monet5/sql_result.c sql/backends/monet5/sql_result.h Branch: protocol Log Message: compressed streams actually working for large results diffs (truncated from 479 to 300 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -3300,17 +3300,8 @@ main(int argc, char **argv) } if (protocol) { - if (strcasecmp(protocol, "prot9") == 0) { - mapi_set_protocol(mid, prot9); - } - else if (strcasecmp(protocol, "prot10") == 0) { - mapi_set_protocol(mid, prot10); - } - else if (strcasecmp(protocol, "prot10compressed") == 0) { - mapi_set_protocol(mid, prot10compressed); - } - else { - fprintf(stderr, "invalid protocol name '%s'\n", protocol); + if (mapi_set_protocol(mid, protocol) != 0) { + fprintf(stderr, "%s\n", mapi_error_str(mid)); } } diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -5540,8 +5540,22 @@ mapi_get_active(Mapi mid) return mid->active; } -void mapi_set_protocol(Mapi mid, protocol_version prot) { - mid->protocol = prot; +MapiMsg mapi_set_protocol(Mapi mid, const char* protocol) { + if (strcasecmp(protocol, "prot9") == 0) { + mid->protocol = prot9; + } + else if (strcasecmp(protocol, "prot10") == 0) { + mid->protocol = prot10; + } + else if (strcasecmp(protocol, "prot10compressed") == 0) { + mid->protocol = prot10compressed; + } + else { + mapi_setError(mid, "invalid protocol name", "mapi_set_protocol", MERROR); + return -1; + } + + return 0; } void mapi_set_blocksize(Mapi mid, size_t blocksize) { diff --git a/clients/mapilib/mapi.h b/clients/mapilib/mapi.h --- a/clients/mapilib/mapi.h +++ b/clients/mapilib/mapi.h @@ -129,13 +129,6 @@ typedef struct { /* used by MAPI_DATETI unsigned int fraction; /* in 1000 millionths of a second (10e-9) */ } MapiDateTime; -typedef enum { - protauto = 0, - prot9 = 1, - prot10 = 2, - prot10compressed = 3, -} protocol_version; - /* connection-oriented functions */ mapi_export Mapi mapi_mapi(const char *host, int port, const char *username, const char *password, const char *lang, const char *dbname); mapi_export Mapi mapi_mapiuri(const char *url, const char *user, const char *pass, const char *lang); @@ -240,7 +233,7 @@ mapi_export char *mapi_quote(const char mapi_export char *mapi_unquote(char *msg); mapi_export MapiHdl mapi_get_active(Mapi mid); -mapi_export void mapi_set_protocol(Mapi mid, protocol_version prot); +mapi_export MapiMsg mapi_set_protocol(Mapi mid, const char* prot); mapi_export void mapi_set_blocksize(Mapi mid, size_t blocksize); #ifdef _MSC_VER diff --git a/common/stream/stream.c b/common/stream/stream.c --- a/common/stream/stream.c +++ b/common/stream/stream.c @@ -3977,6 +3977,7 @@ typedef struct bs2 { size_t nr; /* how far we got in buf */ size_t itotal; /* amount available in current read block */ size_t bufsiz; + size_t readpos; compression_method comp; char *compbuf; size_t compbufsiz; @@ -4047,6 +4048,7 @@ bs2_write(stream *ss, const void *buf, s s->nr += n; todo -= n; buf = ((const char *) buf + n); + /* block is full, write it to the stream */ if (s->nr == s->bufsiz) { #ifdef BSTREAM_DEBUG @@ -4063,17 +4065,15 @@ bs2_write(stream *ss, const void *buf, s } #endif - - /* block is full, write it to the stream */ - writelen = s->nr; blksize = s->nr; writebuf = s->buf; if (s->comp == COMPRESSION_SNAPPY) { + snappy_status ret; size_t compressed_length = s->compbufsiz; - if (snappy_compress(s->buf, s->nr, s->compbuf, &compressed_length) != SNAPPY_OK) { -
MonetDB: protocol - new result set first part and mapi read_line...
Changeset: 8155970591cd for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8155970591cd Modified Files: clients/mapilib/mapi.c common/stream/stream.c common/stream/stream.h sql/backends/monet5/sql_result.c sql/include/sql_query.h Branch: protocol Log Message: new result set first part and mapi read_line cleanup diffs (truncated from 430 to 300 lines): diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -876,9 +876,7 @@ struct MapiRowBuf { struct BlockCache { char *buf; int lim; - int nxt; int end; - int eos;/* end of sequence */ }; @@ -1894,18 +1892,12 @@ mapi_new(void) mid->redircnt = 0; mid->redirmax = 10; mid->tracelog = NULL; - mid->blk.eos = 0; - mid->blk.buf = malloc(BLOCK + 1); + mid->blk.buf = malloc(BLOCK); if (mid->blk.buf == NULL) { mapi_destroy(mid); return NULL; } - mid->blk.buf[BLOCK] = 0; - mid->blk.buf[0] = 0; - mid->blk.nxt = 0; - mid->blk.end = 0; mid->blk.lim = BLOCK; - mid->first = NULL; return mid; @@ -2558,7 +2550,7 @@ mapi_reconnect(Mapi mid) /* consume server challenge */ len = mnstr_read_block(mid->from, buf, 1, BLOCK); - check_stream(mid, mid->from, "Connection terminated while starting", "mapi_reconnect", (mid->blk.eos = 1, mid->error)); + check_stream(mid, mid->from, "Connection terminated while starting", "mapi_reconnect", mid->error); assert(len < BLOCK); buf[len] = 0; @@ -3491,82 +3483,32 @@ mapi_param_store(MapiHdl hdl) static char * read_line(Mapi mid) { - char *reply; - char *nl; - char *s;/* from where to search for newline */ - + int ret = 0; if (mid->active == NULL) return 0; - - /* check if we need to read more blocks to get a new line */ - mid->blk.eos = 0; - s = mid->blk.buf + mid->blk.nxt; - while ((nl = strchr(s, '\n')) == NULL && !mid->blk.eos) { - ssize_t len; - - if (mid->blk.lim - mid->blk.end < BLOCK) { - int len; - - len = mid->blk.lim; - if (mid->blk.nxt <= BLOCK) { - /* extend space */ - len += BLOCK; + mid->blk.end = 0; + do { + if ((mid->blk.end + 1) == mid->blk.lim) { + REALLOC(mid->blk.buf, mid->blk.lim + BLOCK); + if (!mid->blk.buf) { + return 0; } - REALLOC(mid->blk.buf, len + 1); - if (mid->blk.nxt > 0) { - memmove(mid->blk.buf, mid->blk.buf + mid->blk.nxt, mid->blk.end - mid->blk.nxt + 1); - mid->blk.end -= mid->blk.nxt; - mid->blk.nxt = 0; + mid->blk.lim += BLOCK; + } + /* mid->from is **always** buffered, so no point in rolling an additional cache on top */ + if ((ret = mnstr_readChr(mid->from, mid->blk.buf + mid->blk.end)) != 1) { + if (ret == 0) { + mid->blk.buf[0] = PROMPTBEG; + mid->blk.buf[1] = '\n'; + mid->blk.buf[2] = 0; + return mid->blk.buf; } - mid->blk.lim = len; + return 0; } - - s = mid->blk.buf + mid->blk.end; - - /* fetch one more block */ - if (mid->trace == MAPI_TRACE) - printf("fetch next block: start at:%d\n", mid->blk.end); - len = mnstr_read(mid->from, mid->blk.buf + mid->blk.end, 1, BLOCK); - check_stream(mid, mid->from, "Connection terminated during read line", "read_line", (mid->blk.eos = 1, (char *) 0)); - if (mid->tracelog) { - mapi_log_header(mid, "R"); - mnstr_write(mid->tracelog, mid->blk.buf + mid->blk.end, 1, len); - mnstr_flush(mid->tracelog); - } - mid->blk.buf[mid->blk.end + len] = 0; - if (mid->trace == MAPI_TRACE) { - printf("got next block: length:" SSZFMT "\n", len); - printf("text:%s\n", mid->blk.buf + mid->blk.end); - } - if (len == 0) { /* add prompt */ - if (mid->blk.end > mid->blk.nxt) { - /* add fake newline since newline was -* missing from server */ - n
MonetDB: protocol - protobuf prototype and bstream zapping
Changeset: 9002fbfe0dac for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9002fbfe0dac Modified Files: bootstrap common/Makefile.ag configure.ag monetdb5/mal/Makefile.ag monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_debugger.c monetdb5/mal/mal_import.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_parser.c monetdb5/mal/mal_parser.h monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/remote.c monetdb5/optimizer/opt_pipes.c sql/backends/monet5/sql_scenario.c sql/server/sql_mvc.c sql/server/sql_mvc.h sql/server/sql_scan.c sql/server/sql_scan.h Branch: protocol Log Message: protobuf prototype and bstream zapping diffs (truncated from 1067 to 300 lines): diff --git a/bootstrap b/bootstrap --- a/bootstrap +++ b/bootstrap @@ -59,3 +59,5 @@ autoconf if [ -f buildtools/conf/install-sh -a ! -x buildtools/conf/install-sh ]; then chmod +x buildtools/conf/install-sh fi + +protoc-c common/protobuf/mhapi.proto --c_out common/protobuf diff --git a/common/Makefile.ag b/common/Makefile.ag --- a/common/Makefile.ag +++ b/common/Makefile.ag @@ -4,4 +4,4 @@ # # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. -SUBDIRS = stream options utils +SUBDIRS = stream options utils protobuf diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -1641,6 +1641,51 @@ AM_CONDITIONAL([HAVE_LIBSNAPPY], [test x +dnl check for protobuf-c serialization library +org_have_protobuf=auto +have_protobuf=$org_have_protobuf +protobuf_CFLAGS="" +protobuf_LIBS="" +AC_ARG_WITH([protobuf], + [AS_HELP_STRING([--with-protobuf=DIR], + [protobuf library is installed in DIR])], + [have_protobuf="$withval"]) + +AS_CASE(["$have_protobuf"], + [yes|no|auto], [], + [ + protobuf_CFLAGS="-I$withval/include" + protobuf_LIBS="-L$withval/lib"]) + +AC_MSG_CHECKING([for protobuf]) +AS_VAR_IF([have_protobuf], [no], [], [ + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $protobuf_CFLAGS" + save_LIBS="$LIBS" + LIBS="$LIBS $protobuf_LIBS -lprotobuf-c" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include +@%:@include ]], [[(void)protobuf_c_version_number();]])], + protobuf_LIBS="$protobuf_LIBS -lprotobuf-c", + [ AS_VAR_IF([have_protobuf], [auto], [], [ + AC_MSG_ERROR([protobuf library not found])]) + have_protobuf=no + why_have_protobuf="(protobuf library not found)" ]) + LIBS="$save_LIBS" + CPPFLAGS="$save_CPPFLAGS"]) + +AS_VAR_IF([have_protobuf], [no], [], [ + AC_DEFINE([HAVE_LIBPROTOBUF], 1, [Define if you have the protobuf library]) + AC_MSG_RESULT([yes: $protobuf_LIBS])], [ + protobuf_CFLAGS="" + protobuf_LIBS="" + AC_MSG_RESULT([no])]) + +AC_SUBST([protobuf_CFLAGS]) +AC_SUBST([protobuf_LIBS]) +AM_CONDITIONAL([HAVE_LIBPROTOBUF], [test x$have_protobuf != xno]) + + + DL_LIBS="" AC_CHECK_LIB([dl], [dlopen], [DL_LIBS="-ldl" ] ) AC_SUBST([DL_LIBS]) @@ -3284,6 +3329,7 @@ for comp in \ 'pcre ' \ 'perl ' \ 'proj ' \ + 'protobuf ' \ 'pthread ' \ 'python2 ' \ 'python3 ' \ diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag --- a/monetdb5/mal/Makefile.ag +++ b/monetdb5/mal/Makefile.ag @@ -6,6 +6,7 @@ INCLUDES = ../../common/options \ ../../common/stream \ + ../../common/protobuf \ ../../common/utils \ ../../clients/mapilib \ ../../gdk \ @@ -37,7 +38,6 @@ lib_mal = { mal_namespace.c mal_namespace.h \ mal_parser.c mal_parser.h \ mal_profiler.c mal_profiler.h \ - mal_readline.c mal_readline.h \ mal_resolve.c mal_resolve.h \ mal_sabaoth.c mal_sabaoth.h \ mal_scenario.c mal_scenario.h \ diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c --- a/monetdb5/mal/mal_client.c +++ b/monetdb5/mal/mal_client.c @@ -42,7 +42,6 @@ /* (author) M.L. Kersten */ #include "monetdb_config.h" #include "mal_client.h" -#include "mal_readline.h" #include "mal_import.h" #include "mal_parser.h" #include "mal_namespace.h" @@ -94,7 +93,7 @@ MCinit(void) } int -MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt) +MCpushClientInput(Client c, stream *new_input, int listing, char *prompt) { ClientInput *x = (ClientInput *) GDKmalloc(sizeof(ClientInput)); if (x == 0) @@ -119,7 +118,7 @@ MCpopClientInput(Client c) Clien
MonetDB: protocol - proto stuff
Changeset: d5d332081c12 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d5d332081c12 Added Files: common/protobuf/Makefile.ag common/protobuf/messages.c common/protobuf/messages.h common/protobuf/mhapi.proto Modified Files: monetdb5/modules/mal/mal_mapi.c Branch: protocol Log Message: proto stuff diffs (truncated from 313 to 300 lines): diff --git a/common/protobuf/Makefile.ag b/common/protobuf/Makefile.ag new file mode 100644 --- /dev/null +++ b/common/protobuf/Makefile.ag @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. + +## Process this file with automake to produce Makefile.in + +MTSAFE + +INCLUDES = ../stream \ + $(snappy_CFLAGS) \ + $(protobuf_CFLAGS) + +lib_protobuf = { + SOURCES = messages.c messages.h mhapi.pb-c.c mhapi.pb-c.h + LIBS = $(snappy_LIBS) \ + $(protobuf_LIBS) +} + +headers_common = { + DIR = includedir/monetdb + HEADERS = h + SOURCES = messages.h mhapi.pb-c.h +} diff --git a/common/protobuf/messages.c b/common/protobuf/messages.c new file mode 100644 --- /dev/null +++ b/common/protobuf/messages.c @@ -0,0 +1,175 @@ +#include "messages.h" +#ifdef HAVE_LIBSNAPPY +#include // C forever +#endif + +#include /* va_alist.. */ + + +Mhapi__Message* message_read(stream *s, compression_method comp) { + lng len; + mnstr_readLng(s, &len); + return message_read_length(s, comp, len); +} + +Mhapi__Message* message_read_length(stream *s, compression_method comp, lng len) { + Mhapi__Message* ret; + char* read_buf; + + if (len < 1) { + return NULL; + } + read_buf = malloc(len); + if (!read_buf) { + return NULL; + } + if (mnstr_read(s, read_buf, len, 1) != len) { + return NULL; + } + switch(comp) { + case COMPRESSION_NONE: + break; + case COMPRESSION_SNAPPY: +#ifdef HAVE_LIBSNAPPY + { + size_t uncompressed_length; + char *uncompressed_buf; + if (!snappy_uncompressed_length(read_buf, len, &uncompressed_length) == SNAPPY_OK) { + free(read_buf); + return NULL; + } + uncompressed_buf = malloc(uncompressed_length); + if (!uncompressed_buf) { + free(read_buf); + return NULL; + } + if (snappy_uncompress(read_buf, len, uncompressed_buf, &uncompressed_length) != SNAPPY_OK) { + free(read_buf); + free(uncompressed_buf); + return NULL; + } + free(read_buf); + read_buf = uncompressed_buf; + len = uncompressed_length; + } +#else + return NULL; +#endif + } + + ret = (Mhapi__Message*) protobuf_c_message_unpack(&mhapi__message__descriptor, NULL, len, (uint8_t*) read_buf); + free(read_buf); + return ret; +} + +ssize_t message_write(stream *s, compression_method comp, Mhapi__Message *msg) { + lng len = protobuf_c_message_get_packed_size((ProtobufCMessage*) msg); + char* write_buf = malloc(len); + if (!write_buf) { + return -1; + } + if (protobuf_c_message_pack((ProtobufCMessage*) msg, (uint8_t*) write_buf) != (size_t) len) { + return -1; + } + switch(comp) { + case COMPRESSION_NONE: + break; + case COMPRESSION_SNAPPY: +#ifdef HAVE_LIBSNAPPY + { + size_t compressed_length = snappy_max_compressed_length(len); + char *compressed_buf = malloc(compressed_length); + if (!compressed_buf) { + free(write_buf); + return NULL; + } + if (snappy_compress(write_buf, len, compressed_buf, &compressed_length) != SNAPPY_OK) { + free(write_buf); + free(compressed_buf); + return NULL; + } + free(write_buf); + write_buf = compressed_buf; + len = compressed_length; + } +#else + return -1; +#endif + } + + if (!mnstr_writeLng(s, len) || mnstr_write(s, write_buf, len, 1) != len) { + return -1; + } + return len; +} + +void message_send_error(stream *s, protocol_version proto, const char *format, ...) { + char buf[BUFSIZ], *bf = buf; + int i = 0; + va_list ap; + + if (s == NULL) { + return; + } + va_start
MonetDB: protocol - wh
Changeset: a24577108238 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a24577108238 Modified Files: common/protobuf/mhapi.proto Branch: protocol Log Message: wh diffs (10 lines): diff --git a/common/protobuf/mhapi.proto b/common/protobuf/mhapi.proto --- a/common/protobuf/mhapi.proto +++ b/common/protobuf/mhapi.proto @@ -68,4 +68,4 @@ message Message { Warning warning = 6; QueryResult queryresult = 7; } -} \ No newline at end of file +} ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: protocol - Backed out changeset a24577108238
Changeset: 814c5a3fad3b for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=814c5a3fad3b Modified Files: common/protobuf/mhapi.proto Branch: protocol Log Message: Backed out changeset a24577108238 diffs (10 lines): diff --git a/common/protobuf/mhapi.proto b/common/protobuf/mhapi.proto --- a/common/protobuf/mhapi.proto +++ b/common/protobuf/mhapi.proto @@ -68,4 +68,4 @@ message Message { Warning warning = 6; QueryResult queryresult = 7; } -} +} \ No newline at end of file ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: protocol - Backed out changeset d5d332081c12
Changeset: 204f8ef1cfb4 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=204f8ef1cfb4 Removed Files: common/protobuf/Makefile.ag common/protobuf/messages.c common/protobuf/messages.h common/protobuf/mhapi.proto Modified Files: monetdb5/modules/mal/mal_mapi.c Branch: protocol Log Message: Backed out changeset d5d332081c12 diffs (truncated from 313 to 300 lines): diff --git a/common/protobuf/Makefile.ag b/common/protobuf/Makefile.ag deleted file mode 100644 --- a/common/protobuf/Makefile.ag +++ /dev/null @@ -1,25 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. - -## Process this file with automake to produce Makefile.in - -MTSAFE - -INCLUDES = ../stream \ - $(snappy_CFLAGS) \ - $(protobuf_CFLAGS) - -lib_protobuf = { - SOURCES = messages.c messages.h mhapi.pb-c.c mhapi.pb-c.h - LIBS = $(snappy_LIBS) \ - $(protobuf_LIBS) -} - -headers_common = { - DIR = includedir/monetdb - HEADERS = h - SOURCES = messages.h mhapi.pb-c.h -} diff --git a/common/protobuf/messages.c b/common/protobuf/messages.c deleted file mode 100644 --- a/common/protobuf/messages.c +++ /dev/null @@ -1,175 +0,0 @@ -#include "messages.h" -#ifdef HAVE_LIBSNAPPY -#include // C forever -#endif - -#include /* va_alist.. */ - - -Mhapi__Message* message_read(stream *s, compression_method comp) { - lng len; - mnstr_readLng(s, &len); - return message_read_length(s, comp, len); -} - -Mhapi__Message* message_read_length(stream *s, compression_method comp, lng len) { - Mhapi__Message* ret; - char* read_buf; - - if (len < 1) { - return NULL; - } - read_buf = malloc(len); - if (!read_buf) { - return NULL; - } - if (mnstr_read(s, read_buf, len, 1) != len) { - return NULL; - } - switch(comp) { - case COMPRESSION_NONE: - break; - case COMPRESSION_SNAPPY: -#ifdef HAVE_LIBSNAPPY - { - size_t uncompressed_length; - char *uncompressed_buf; - if (!snappy_uncompressed_length(read_buf, len, &uncompressed_length) == SNAPPY_OK) { - free(read_buf); - return NULL; - } - uncompressed_buf = malloc(uncompressed_length); - if (!uncompressed_buf) { - free(read_buf); - return NULL; - } - if (snappy_uncompress(read_buf, len, uncompressed_buf, &uncompressed_length) != SNAPPY_OK) { - free(read_buf); - free(uncompressed_buf); - return NULL; - } - free(read_buf); - read_buf = uncompressed_buf; - len = uncompressed_length; - } -#else - return NULL; -#endif - } - - ret = (Mhapi__Message*) protobuf_c_message_unpack(&mhapi__message__descriptor, NULL, len, (uint8_t*) read_buf); - free(read_buf); - return ret; -} - -ssize_t message_write(stream *s, compression_method comp, Mhapi__Message *msg) { - lng len = protobuf_c_message_get_packed_size((ProtobufCMessage*) msg); - char* write_buf = malloc(len); - if (!write_buf) { - return -1; - } - if (protobuf_c_message_pack((ProtobufCMessage*) msg, (uint8_t*) write_buf) != (size_t) len) { - return -1; - } - switch(comp) { - case COMPRESSION_NONE: - break; - case COMPRESSION_SNAPPY: -#ifdef HAVE_LIBSNAPPY - { - size_t compressed_length = snappy_max_compressed_length(len); - char *compressed_buf = malloc(compressed_length); - if (!compressed_buf) { - free(write_buf); - return NULL; - } - if (snappy_compress(write_buf, len, compressed_buf, &compressed_length) != SNAPPY_OK) { - free(write_buf); - free(compressed_buf); - return NULL; - } - free(write_buf); - write_buf = compressed_buf; - len = compressed_length; - } -#else - return -1; -#endif - } - - if (!mnstr_writeLng(s, len) || mnstr_write(s, write_buf, len, 1) != len) { - return -1; - } - return len; -} - -void message_send_error(stream *s, protocol_version proto, const char *format, ...) { - char buf[BUFSIZ], *bf = buf; - int i = 0; - va_list ap; - - if (s == NULL) { - re
MonetDB: protocol - Backed out changeset 9002fbfe0dac
Changeset: 9786385f19cc for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9786385f19cc Modified Files: bootstrap common/Makefile.ag configure.ag monetdb5/mal/Makefile.ag monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_debugger.c monetdb5/mal/mal_import.c monetdb5/mal/mal_interpreter.c monetdb5/mal/mal_parser.c monetdb5/mal/mal_parser.h monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/modules/mal/Makefile.ag monetdb5/modules/mal/mal_io.c monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/remote.c monetdb5/optimizer/opt_pipes.c sql/backends/monet5/sql_scenario.c sql/server/sql_mvc.c sql/server/sql_mvc.h sql/server/sql_scan.c sql/server/sql_scan.h Branch: protocol Log Message: Backed out changeset 9002fbfe0dac diffs (truncated from 1066 to 300 lines): diff --git a/bootstrap b/bootstrap --- a/bootstrap +++ b/bootstrap @@ -59,5 +59,3 @@ autoconf if [ -f buildtools/conf/install-sh -a ! -x buildtools/conf/install-sh ]; then chmod +x buildtools/conf/install-sh fi - -protoc-c common/protobuf/mhapi.proto --c_out common/protobuf diff --git a/common/Makefile.ag b/common/Makefile.ag --- a/common/Makefile.ag +++ b/common/Makefile.ag @@ -4,4 +4,4 @@ # # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. -SUBDIRS = stream options utils protobuf +SUBDIRS = stream options utils diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -1641,51 +1641,6 @@ AM_CONDITIONAL([HAVE_LIBSNAPPY], [test x -dnl check for protobuf-c serialization library -org_have_protobuf=auto -have_protobuf=$org_have_protobuf -protobuf_CFLAGS="" -protobuf_LIBS="" -AC_ARG_WITH([protobuf], - [AS_HELP_STRING([--with-protobuf=DIR], - [protobuf library is installed in DIR])], - [have_protobuf="$withval"]) - -AS_CASE(["$have_protobuf"], - [yes|no|auto], [], - [ - protobuf_CFLAGS="-I$withval/include" - protobuf_LIBS="-L$withval/lib"]) - -AC_MSG_CHECKING([for protobuf]) -AS_VAR_IF([have_protobuf], [no], [], [ - save_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $protobuf_CFLAGS" - save_LIBS="$LIBS" - LIBS="$LIBS $protobuf_LIBS -lprotobuf-c" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include -@%:@include ]], [[(void)protobuf_c_version_number();]])], - protobuf_LIBS="$protobuf_LIBS -lprotobuf-c", - [ AS_VAR_IF([have_protobuf], [auto], [], [ - AC_MSG_ERROR([protobuf library not found])]) - have_protobuf=no - why_have_protobuf="(protobuf library not found)" ]) - LIBS="$save_LIBS" - CPPFLAGS="$save_CPPFLAGS"]) - -AS_VAR_IF([have_protobuf], [no], [], [ - AC_DEFINE([HAVE_LIBPROTOBUF], 1, [Define if you have the protobuf library]) - AC_MSG_RESULT([yes: $protobuf_LIBS])], [ - protobuf_CFLAGS="" - protobuf_LIBS="" - AC_MSG_RESULT([no])]) - -AC_SUBST([protobuf_CFLAGS]) -AC_SUBST([protobuf_LIBS]) -AM_CONDITIONAL([HAVE_LIBPROTOBUF], [test x$have_protobuf != xno]) - - - DL_LIBS="" AC_CHECK_LIB([dl], [dlopen], [DL_LIBS="-ldl" ] ) AC_SUBST([DL_LIBS]) @@ -3329,7 +3284,6 @@ for comp in \ 'pcre ' \ 'perl ' \ 'proj ' \ - 'protobuf ' \ 'pthread ' \ 'python2 ' \ 'python3 ' \ diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag --- a/monetdb5/mal/Makefile.ag +++ b/monetdb5/mal/Makefile.ag @@ -6,7 +6,6 @@ INCLUDES = ../../common/options \ ../../common/stream \ - ../../common/protobuf \ ../../common/utils \ ../../clients/mapilib \ ../../gdk \ @@ -38,6 +37,7 @@ lib_mal = { mal_namespace.c mal_namespace.h \ mal_parser.c mal_parser.h \ mal_profiler.c mal_profiler.h \ + mal_readline.c mal_readline.h \ mal_resolve.c mal_resolve.h \ mal_sabaoth.c mal_sabaoth.h \ mal_scenario.c mal_scenario.h \ diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c --- a/monetdb5/mal/mal_client.c +++ b/monetdb5/mal/mal_client.c @@ -42,6 +42,7 @@ /* (author) M.L. Kersten */ #include "monetdb_config.h" #include "mal_client.h" +#include "mal_readline.h" #include "mal_import.h" #include "mal_parser.h" #include "mal_namespace.h" @@ -93,7 +94,7 @@ MCinit(void) } int -MCpushClientInput(Client c, stream *new_input, int listing, char *prompt) +MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt) { ClientInput *x = (ClientInput *) GDKmalloc(sizeof(ClientInput)); if (x == 0) @@ -118,7 +119,7 @@ MCpopClientInput(Client c) ClientInpu
MonetDB: protocol - client
Changeset: e79f82d345d7 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e79f82d345d7 Modified Files: clients/mapiclient/mclient.c clients/mapilib/mapi.c sql/backends/monet5/sql_result.c Branch: protocol Log Message: client diffs (185 lines): diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -330,13 +330,16 @@ fetch_line(MapiHdl hdl) static int fetch_row(MapiHdl hdl) { - char *reply; - - do { - if ((reply = fetch_line(hdl)) == NULL) - return 0; - } while (*reply != '[' && *reply != '='); - return mapi_split_line(hdl); + // yeah right. circumvent your own api... +// char *reply; +// +// do { +// if ((reply = fetch_line(hdl)) == NULL) +// return 0; +// } while (*reply != '[' && *reply != '='); +// return mapi_split_line(hdl); +// + return mapi_fetch_row(hdl); } static void diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c --- a/clients/mapilib/mapi.c +++ b/clients/mapilib/mapi.c @@ -826,6 +826,7 @@ struct MapiColumn { int columnlength; int digits; int scale; + void* dataprot10; }; /* information about bound columns */ @@ -938,6 +939,7 @@ struct MapiResultSet { struct MapiColumn *fields; struct MapiRowBuf cache; int commentonly;/* only comments seen so far */ + mapi_int64 rows_read; }; struct MapiStatement { @@ -4010,13 +4012,23 @@ read_into_cache(MapiHdl hdl, int lookahe lng nr_rows; lng nr_cols; lng i; + result = malloc(sizeof(struct MapiResultSet)); + if (!result) { + // TODO: actually set mid->error :) + return mid->error; + } if (!mnstr_readInt(mid->from, &result_set_id) || !mnstr_readLng(mid->from, &nr_rows) || !mnstr_readLng(mid->from, &nr_cols)) { return mid->error; } fprintf(stderr, "result_set_id=%d, nr_rows=%llu, nr_cols=%lld\n", result_set_id, nr_rows, nr_cols); - + result->fieldcnt = nr_cols; + result->row_count = nr_rows; + result->fields = malloc(sizeof(struct MapiColumn) * result->fieldcnt); + result->tableid = result_set_id; + result->querytype = Q_TABLE; + result->tuple_count = 0; for (i = 0; i < nr_cols; i++) { lng col_info_length; @@ -4040,18 +4052,19 @@ read_into_cache(MapiHdl hdl, int lookahe !mnstr_readInt(mid->from, &typelen)) { return mid->error; } - fprintf(stderr, "%lld col_info_length=%lld, table_name=%s, col_name=%s, type_sql_name=%s, type_len=%d\n", i, col_info_length, table_name, col_name, type_sql_name, typelen); + fprintf(stderr, "%lld col_info_length=%lld, table_name=%s, col_name=%s, type_sql_name=%s, type_len=%d\n", + i, col_info_length, table_name, col_name, type_sql_name, typelen); + result->fields[i].columnname = col_name; + result->fields[i].tablename = table_name; + result->fields[i].columntype = type_sql_name; } + hdl->result = result; + hdl->active = result; { - lng nrows = 0; char dummy; // we flush on the other side so this read will always fail mnstr_readChr(mid->from, &dummy); - if (!mnstr_readLng(mid->from, &nrows)) { - return mid->error; - } - fprintf(stderr, "nrows=%llu\n", nrows); } @@ -5276,6 +5289,27 @@ mapi_fetch_row(MapiHdl hdl) int n; struct MapiResultSet *result; + if (hdl->mid->protocol == prot10) { + result = hdl->result; + result->rows_read++; + // do we have any rows in our cache + if (result->rows_read > result->tuple_count && result->rows_read < result->row_count) { + // read block from socket + lng nrows = 0; +