from:"Hannes Muehleisen"

MonetDB: Jul2017 - no warnings

2017-10-17 Thread Hannes Muehleisen

Changeset: e5559912c6e8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e5559912c6e8
Modified Files:
clients/R/Tests/deps-install.R
clients/R/Tests/install.R
Branch: Jul2017
Log Message:

no warnings


diffs (17 lines):

diff --git a/clients/R/Tests/deps-install.R b/clients/R/Tests/deps-install.R
--- a/clients/R/Tests/deps-install.R
+++ b/clients/R/Tests/deps-install.R
@@ -1,4 +1,5 @@
 # autoinstall DBI and digest, we need those to install MonetDB.R
+options(warn = -1)
 packages_required <- c("digest", "DBI")
 
 cat("#~BeginProfilingOutput~#\n", file=stderr())
diff --git a/clients/R/Tests/install.R b/clients/R/Tests/install.R
--- a/clients/R/Tests/install.R
+++ b/clients/R/Tests/install.R
@@ -1,3 +1,4 @@
+options(warn = -1)
 cat("#~BeginProfilingOutput~#\n", file=stderr())
 cat("#~BeginProfilingOutput~#\n", file=stdout())
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jul2017 - making r tests run again

2017-10-16 Thread Hannes Muehleisen

Changeset: 90e068ebfcf6 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90e068ebfcf6
Modified Files:
clients/R/Tests/All
clients/R/Tests/copy_into_fwf.stable.out
clients/R/Tests/dbi.R
clients/R/Tests/dbi.stable.err
clients/R/Tests/deps-install.R
clients/R/Tests/deps-test.R
clients/R/Tests/dplyr-flights.R
clients/R/Tests/dplyr-flights.reqtests
clients/R/Tests/dplyr.R
clients/R/Tests/install.R
Branch: Jul2017
Log Message:

making r tests run again


diffs (182 lines):

diff --git a/clients/R/Tests/All b/clients/R/Tests/All
--- a/clients/R/Tests/All
+++ b/clients/R/Tests/All
@@ -1,8 +1,8 @@
 HAVE_LIBR?deps-install
 HAVE_LIBR?install
 HAVE_LIBR?deps-test
-HAVE_LIBR_PERL?dbi
-HAVE_LIBR?dplyr
+HAVE_LIBR?dbi
+#HAVE_LIBR?dplyr too slow
 HAVE_LIBR?dplyr-flights
 HAVE_LIBR?copy_into_fwf
 HAVE_LIBR_WIN32?dbapply
diff --git a/clients/R/Tests/copy_into_fwf.stable.out 
b/clients/R/Tests/copy_into_fwf.stable.out
--- a/clients/R/Tests/copy_into_fwf.stable.out
+++ b/clients/R/Tests/copy_into_fwf.stable.out
@@ -26,6 +26,8 @@ Ready.
 
 
   SQL  CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp 
DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE 
PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, 
gear DOUBLE PRECISION, carb DOUBLE PRECISION)
+  ROWS Fetched: 0 [complete]
+   Changed: NA
 mpg cyl  disp  hp dratwt  qsec vs am gear carb
 1  21.0   6 160.0 110 3.90 2.620 16.46  0  144
 2  21.0   6 160.0 110 3.90 2.875 17.02  0  144
diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R
--- a/clients/R/Tests/dbi.R
+++ b/clients/R/Tests/dbi.R
@@ -198,7 +198,7 @@ stopifnot(identical(1L, as.integer(dbGet
 # reserved words in data frame column names
 stopifnot(dbIsValid(conn))
 dbBegin(conn)
-dbWriteTable(conn, "evilt", data.frame(year=42, month=12, day=24, 
some.dot=12), transaction=F)
+dbWriteTable(conn, "evilt", data.frame(year=42, month=12, day=24, some.dot=12))
 stopifnot(dbExistsTable(conn, "evilt"))
 dbRollback(conn)
 
@@ -208,7 +208,7 @@ dbBegin(conn)
 data(api, package="survey")
 x <- apiclus1
 x$idkey <- seq( nrow( x ) )
-dbWriteTable( conn , 'x' , x , transaction=F)
+dbWriteTable( conn , 'x' , x )
 stopifnot(dbExistsTable(conn, "x"))
 dbRollback(conn)
 
@@ -217,7 +217,7 @@ stopifnot(!is.null(dbGetQuery(conn, "SEL
 
 #non-standard dbwritetable use
 dbBegin(conn)
-dbWriteTable(conn, "vectable", 1:1000, transaction=F)
+dbWriteTable(conn, "vectable", 1:1000)
 stopifnot(dbExistsTable(conn, "vectable"))
 dbRollback(conn)
 
diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err
--- a/clients/R/Tests/dbi.stable.err
+++ b/clients/R/Tests/dbi.stable.err
@@ -33,19 +33,16 @@ stderr of test 'dbi` in directory 'clien
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Warning message:
 'MonetDBLite::isIdCurrent' is deprecated.
-Use 'dbIsValid' instead.
+Use 'use dbIsValid() instead' instead.
 See help("Deprecated") 
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
-Treating character vector parameter as file name(s) for monetdb.read.csv()
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
-Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
-Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "some.dot" contain uppercase or reserved SQL characters and 
need(s) to be quoted in queries.
 Identifier(s) "year", "month", "day" are reserved SQL keywords and need(s) to 
be quoted in queries.
 Identifier(s) "sch.wide", "comp.imp", "yr.rnd", "acs.k3", "acs.46", 
"acs.core", "pct.resp", "not.hsg",

MonetDB: Dec2016 - disabled r tests for now because of their hig...

2017-03-27 Thread Hannes Muehleisen

Changeset: e798e9acbc37 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e798e9acbc37
Modified Files:
clients/R/Tests/All
Branch: Dec2016
Log Message:

disabled r tests for now because of their high volatility


diffs (20 lines):

diff --git a/clients/R/Tests/All b/clients/R/Tests/All
--- a/clients/R/Tests/All
+++ b/clients/R/Tests/All
@@ -1,8 +1,8 @@
-HAVE_LIBR?deps-install
-HAVE_LIBR?install
-HAVE_LIBR?deps-test
-HAVE_LIBR?dbi
-HAVE_LIBR?dplyr
-HAVE_LIBR?dplyr-flights
-HAVE_LIBR?copy_into_fwf
-HAVE_LIBR_WIN32?dbapply
+#HAVE_LIBR?deps-install
+#HAVE_LIBR?install
+#HAVE_LIBR?deps-test
+#HAVE_LIBR?dbi
+#HAVE_LIBR?dplyr
+#HAVE_LIBR?dplyr-flights
+#HAVE_LIBR?copy_into_fwf
+#HAVE_LIBR_WIN32?dbapply
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - more unchecked mallocs

2017-03-13 Thread Hannes Muehleisen

Changeset: 5d806584f589 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d806584f589
Modified Files:
sql/backends/monet5/sql_statement.c
sql/storage/store_dependency.c
Branch: default
Log Message:

more unchecked mallocs


diffs (38 lines):

diff --git a/sql/backends/monet5/sql_statement.c 
b/sql/backends/monet5/sql_statement.c
--- a/sql/backends/monet5/sql_statement.c
+++ b/sql/backends/monet5/sql_statement.c
@@ -2306,6 +2306,7 @@ stmt_output(backend *be, stmt *lst)
const char *ntn = sql_escape_ident(tn);
const char *nsn = sql_escape_ident(sn);
size_t fqtnl = strlen(ntn) + 1 + strlen(nsn) + 1;
+   // FIXME unchecked_malloc NEW_ARRAY can return NULL
char *fqtn = NEW_ARRAY(char, fqtnl);
 
snprintf(fqtn, fqtnl, "%s.%s", nsn, ntn);
diff --git a/sql/storage/store_dependency.c b/sql/storage/store_dependency.c
--- a/sql/storage/store_dependency.c
+++ b/sql/storage/store_dependency.c
@@ -111,6 +111,7 @@ sql_trans_get_dependencies(sql_trans* tr
for(rid = table_funcs.rids_next(rs); rid != oid_nil; rid = 
table_funcs.rids_next(rs)) {
v = table_funcs.column_find_value(tr, tri_id, rid);
list_append(dep_list, v);
+   // FIXME unchecked_malloc MNEW can return NULL
v = MNEW(sht);
*(sht*)v = depend_type;
list_append(dep_list, v);
@@ -191,6 +192,7 @@ sql_trans_schema_user_dependencies(sql_t
for(rid = table_funcs.rids_next(users); rid != oid_nil; rid = 
table_funcs.rids_next(users)) {
v = table_funcs.column_find_value(tr, auth_id, rid);
list_append(l,v);
+   // FIXME unchecked_malloc MNEW can return NULL
v = MNEW(sht);
*(sht*)v = type;
list_append(l,v);
@@ -216,6 +218,7 @@ sql_trans_owner_schema_dependencies(sql_
for(rid = table_funcs.rids_next(rs); rid != oid_nil; rid = 
table_funcs.rids_next(rs)) {
v = table_funcs.column_find_value(tr, schema_id, rid);
list_append(l, v);
+   // FIXME unchecked_malloc MNEW can return NULL
v = MNEW(sht);
*(sht*)v = type;
list_append(l,v);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - fix compilation

2017-03-13 Thread Hannes Muehleisen

Changeset: 43013a1ede09 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=43013a1ede09
Modified Files:
monetdb5/modules/mal/txtsim.c
Branch: default
Log Message:

fix compilation


diffs (12 lines):

diff --git a/monetdb5/modules/mal/txtsim.c b/monetdb5/modules/mal/txtsim.c
--- a/monetdb5/modules/mal/txtsim.c
+++ b/monetdb5/modules/mal/txtsim.c
@@ -27,7 +27,7 @@
if (b) {\
   if (ATOMextern(t)) {\
  *(ptr*) res = (ptr) ATOMnil(t);\
-   // FIXME unchecked_malloc ATOMnil can return NULL \
+   /* FIXME unchecked_malloc ATOMnil can return NULL */ \
   } else {\
  memcpy(res, ATOMnilptr(t), ATOMsize(t));\
   }\
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Added markers to invocations of GDKmalloc and...

2017-03-13 Thread Hannes Muehleisen

Changeset: 34747c9d31c1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=34747c9d31c1
Modified Files:
gdk/gdk_logger.c
gdk/gdk_value.c
geom/monetdb5/geom.c
monetdb5/mal/mal_builder.c
monetdb5/modules/atoms/blob.c
monetdb5/modules/atoms/inet.c
monetdb5/modules/atoms/uuid.c
monetdb5/modules/atoms/xml.c
monetdb5/modules/mal/calc.c
monetdb5/modules/mal/tablet.c
monetdb5/modules/mal/txtsim.c
monetdb5/optimizer/opt_mergetable.c
sql/backends/monet5/UDF/pyapi/emit.c
sql/backends/monet5/mal_backend.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_statement.c
sql/backends/monet5/sql_statistics.c
sql/backends/monet5/sql_upgrades.c
sql/backends/monet5/vaults/fits/fits.c
sql/backends/monet5/vaults/netcdf/netcdf.c
sql/common/sql_keyword.c
sql/common/sql_mem.c
sql/common/sql_string.c
sql/server/rel_updates.c
sql/server/sql_atom.c
sql/server/sql_mvc.c
sql/server/sql_qc.c
sql/server/sql_scan.c
sql/storage/bat/bat_table.c
sql/storage/bat/res_table.c
sql/storage/store.c
sql/storage/store_sequence.c
tools/mserver/mserver5.c
Branch: default
Log Message:

Added markers to invocations of GDKmalloc and friends where return value is not 
checked. (//FIXME unchecked_malloc ...)


diffs (truncated from 803 to 300 lines):

diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -345,7 +345,10 @@ log_read_updates(logger *lg, trans *tr, 
if (tt < TYPE_str)
tv = lg->buf;
else if (tt > TYPE_str)
+   // FIXME unchecked_malloc ATOMnil can return NULL
+
tv = ATOMnil(tt);
+
assert(l->nr <= (lng) BUN_MAX);
if (l->flag == LOG_UPDATE) {
uid = COLnew(0, ht, (BUN) l->nr, PERSISTENT);
@@ -372,6 +375,8 @@ log_read_updates(logger *lg, trans *tr, 
}
} else {
void *(*rh) (ptr, stream *, size_t) = ht == TYPE_void ? 
BATatoms[TYPE_oid].atomRead : BATatoms[ht].atomRead;
+   // FIXME unchecked_malloc ATOMnil can return NULL
+
void *hv = ATOMnil(ht);
 
for (; l->nr > 0; l->nr--) {
diff --git a/gdk/gdk_value.c b/gdk/gdk_value.c
--- a/gdk/gdk_value.c
+++ b/gdk/gdk_value.c
@@ -143,6 +143,7 @@ VALcopy(ValPtr d, const ValRecord *s)
if (!ATOMextern(s->vtype)) {
*d = *s;
} else if (s->val.pval == 0) {
+   // FIXME unchecked_malloc ATOMnil can return NULL
d->val.pval = ATOMnil(s->vtype);
d->vtype = s->vtype;
} else if (s->vtype == TYPE_str) {
diff --git a/geom/monetdb5/geom.c b/geom/monetdb5/geom.c
--- a/geom/monetdb5/geom.c
+++ b/geom/monetdb5/geom.c
@@ -1745,7 +1745,7 @@ dumpPointsPoint(BAT *idBAT, BAT *geomBAT
str err = MAL_SUCCEED;
 
(*lvl)++;
-
+   // FIXME unchecked_malloc GDKmalloc can return NULL
newPath = GDKmalloc(pathLength + lvlDigitsNum + 1);
sprintf(newPath, "%s%u", path, *lvl);
 
@@ -1805,7 +1805,7 @@ dumpPointsPolygon(BAT *idBAT, BAT *geomB
throw(MAL, "geom.DumpPoints", "GEOSGetExteriorRing failed");
 
(*lvl)++;
-
+   // FIXME unchecked_malloc GDKmalloc can return NULL
newPath = GDKmalloc(pathLength + lvlDigitsNum + extraLength + 1);
sprintf(newPath, "%s%u%s", path, *lvl, extraStr);
 
@@ -1858,6 +1858,7 @@ dumpPointsMultiGeometry(BAT *idBAT, BAT 
multiGeometry = GEOSGetGeometryN(geosGeometry, i);
lvl++;
 
+   // FIXME unchecked_malloc GDKmalloc can return NULL
newPath = GDKmalloc(pathLength + lvlDigitsNum + extraLength + 
1);
sprintf(newPath, "%s%u%s", path, lvl, extraStr);
 
@@ -2165,6 +2166,7 @@ wkbFROMSTR_withSRID(char *geomWKT, int *
if (strncasecmp(geomWKT, polyhedralSurface, strlen(polyhedralSurface)) 
== 0) {
size_t sizeOfInfo = strlen(geomWKT) - strlen(polyhedralSurface);
geomWKT_original = geomWKT;
+   // FIXME unchecked_malloc GDKmalloc can return NULL
geomWKT = GDKmalloc(sizeOfInfo + strlen(multiPolygon) + 1);
strcpy(geomWKT, multiPolygon);
memcpy(geomWKT + strlen(multiPolygon), 
_original[strlen(polyhedralSurface)], sizeOfInfo);
@@ -2219,7 +2221,7 @@ wkbaFROMSTR_withSRID(char *fromStr, int 
//read the number of items from the beginning of the string
memcpy(, fromStr, sizeof(int));
skipBytes += sizeof(int);
-
+

MonetDB: default - more unchecked malloc failures

2017-03-10 Thread Hannes Muehleisen

Changeset: 2fcd5e98b30e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2fcd5e98b30e
Modified Files:
monetdb5/mal/mal_builder.c
monetdb5/optimizer/opt_deadcode.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_support.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_gencode.c
Branch: default
Log Message:

more unchecked malloc failures


diffs (108 lines):

diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c
--- a/monetdb5/mal/mal_builder.c
+++ b/monetdb5/mal/mal_builder.c
@@ -85,7 +85,7 @@ newComment(MalBlkPtr mb, const char *val
InstrPtr q = newInstruction(mb, NULL, NULL);
ValRecord cst;
 
-   if ( q == NULL)
+   if (q == NULL)
return NULL;
q->token = REMsymbol;
q->barrier = 0;
diff --git a/monetdb5/optimizer/opt_deadcode.c 
b/monetdb5/optimizer/opt_deadcode.c
--- a/monetdb5/optimizer/opt_deadcode.c
+++ b/monetdb5/optimizer/opt_deadcode.c
@@ -34,8 +34,10 @@ OPTdeadcodeImplementation(Client cntxt, 
 
limit = mb->stop;
slimit = mb->ssize;
-   if (newMalBlkStmt(mb, mb->ssize) < 0)
+   if (newMalBlkStmt(mb, mb->ssize) < 0) {
+   actions = -1;
goto wrapup;
+   }
 
// Calculate the instructions in which a variable is used.
// Variables can be used multiple times in an instruction.
diff --git a/monetdb5/optimizer/opt_evaluate.c 
b/monetdb5/optimizer/opt_evaluate.c
--- a/monetdb5/optimizer/opt_evaluate.c
+++ b/monetdb5/optimizer/opt_evaluate.c
@@ -179,7 +179,11 @@ OPTevaluateImplementation(Client cntxt, 
profiler = malProfileMode;  /* we don't trace it */
malProfileMode = 0;
if ( env == NULL) {
-   env = prepareMALstack(mb,  2 * mb->vsize );
+   env = prepareMALstack(mb,  2 * mb->vsize);
+   if (!env) {
+   actions = -1;
+   goto wrapup;
+   }
env->keepAlive = TRUE;
}
msg = reenterMAL(cntxt, mb, i, i + 1, env);
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -144,9 +144,11 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
if (msg) {
str place = getExceptionPlace(msg);
str nmsg = 
createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg));
-   GDKfree(place);
-   GDKfree(msg);
-   msg = nmsg;
+   if (nmsg && place) {
+   GDKfree(msg);
+   msg = nmsg;
+   GDKfree(place);
+   }
goto wrapup;
}
if (cntxt->mode == FINISHCLIENT)
@@ -158,11 +160,14 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
 
 wrapup:
/* Keep the total time spent on optimizing the plan for inspection */
-   if( actions){
-   mb->optimize= GDKusec() - clk;
-   snprintf(buf, 256, "%-20s actions=%2d time=" LLFMT " usec", 
"total",actions, mb->optimize);
+   if(actions > 0 && msg == MAL_SUCCEED){
+   mb->optimize = GDKusec() - clk;
+   snprintf(buf, 256, "%-20s actions=%2d time=" LLFMT " usec", 
"total", actions, mb->optimize);
newComment(mb, buf);
}
+   if (msg != MAL_SUCCEED) {
+   mb->errors++;
+   }
if (cnt >= mb->stop)
throw(MAL, "optimizer.MALoptimizer", OPTIMIZER_CYCLE);
return msg;
diff --git a/sql/backends/monet5/sql_execute.c 
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -319,8 +319,9 @@ SQLrun(Client c, backend *be, mvc *m){
msg = SQLoptimizeQuery(c, mb);
mb->keephistory = FALSE;
 
-   if( mb->errors){
-   freeMalBlk(mb);
+   if (mb->errors){
+   //freeMalBlk(mb);
+   // mal block might be so broken free causes segfault
return msg;
}
 
diff --git a/sql/backends/monet5/sql_gencode.c 
b/sql/backends/monet5/sql_gencode.c
--- a/sql/backends/monet5/sql_gencode.c
+++ b/sql/backends/monet5/sql_gencode.c
@@ -662,6 +662,10 @@ backend_call(backend *be, Client c, cq *
MalBlkPtr mb = c->curprg->def;
 
q = newStmt(mb, userRef, cq->name);
+

MonetDB: default - more malloc fail checks and re-enabled test case

2017-03-10 Thread Hannes Muehleisen

Changeset: 7c0e4baa6fed for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7c0e4baa6fed
Modified Files:
monetdb5/mal/mal_instruction.c
monetdb5/optimizer/opt_wrapper.c
sql/backends/monet5/sql_execute.c
sql/test/malloc_fail/Tests/All
sql/test/malloc_fail/Tests/mallocs.SQL.py
Branch: default
Log Message:

more malloc fail checks and re-enabled test case


diffs (110 lines):

diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -263,13 +263,17 @@ copyMalBlk(MalBlkPtr old)
// copy all variable records
for (i = 0; i < old->vtop; i++) {
mb->var[i]=  old->var[i];
-   VALcopy(&(mb->var[i].value), &(old->var[i].value));
+   if (!VALcopy(&(mb->var[i].value), &(old->var[i].value))) {
+   GDKfree(mb);
+   GDKerror("copyMalBlk:" MAL_MALLOC_FAIL);
+   return NULL;
+   }
}
 
mb->stmt = (InstrPtr *) GDKzalloc(sizeof(InstrPtr) * old->ssize);
 
if (mb->stmt == NULL) {
-   GDKfree(mb->var);
+   GDKfree(mb->var); // this leaks strings in var
GDKfree(mb);
GDKerror("copyMalBlk:" MAL_MALLOC_FAIL);
return NULL;
@@ -278,10 +282,20 @@ copyMalBlk(MalBlkPtr old)
mb->stop = old->stop;
mb->ssize = old->ssize;
assert(old->stop < old->ssize);
-   for (i = 0; i < old->stop; i++)
+   for (i = 0; i < old->stop; i++) {
mb->stmt[i] = copyInstruction(old->stmt[i]);
-
+   if(!mb->stmt[i]) {
+   GDKfree(mb);
+   GDKerror("copyMalBlk:" MAL_MALLOC_FAIL);
+   return NULL;
+   }
+   }
mb->help = old->help ? GDKstrdup(old->help) : NULL;
+   if (old->help && !mb->help) {
+   GDKfree(mb);
+   GDKerror("copyMalBlk:" MAL_MALLOC_FAIL);
+   return NULL;
+   }
strncpy(mb->binding,  old->binding, IDLENGTH);
mb->errors = old->errors;
mb->tag = old->tag;
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -146,7 +146,7 @@ str OPTwrapper (Client cntxt, MalBlkPtr 
for (i=0; codes[i].nme; i++)
if (strcmp(codes[i].nme, optimizer) == 0){
actions = (int)(*(codes[i].fcn))(cntxt, mb, stk, 0);
-   if (actions < 1) {
+   if (actions < 0) {
throw(MAL, optimizer, "Error in optimizer %s", 
optimizer);
}
break;  
diff --git a/sql/backends/monet5/sql_execute.c 
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -271,6 +271,9 @@ SQLrun(Client c, backend *be, mvc *m){
return createException(PARSE, "SQLparser", "%s", m->errstr);
// locate and inline the query template instruction
mb = copyMalBlk(c->curprg->def);
+   if (!mb) {
+   throw(SQL, "sql.prepare", "Out of memory");
+   }
mb->history = c->curprg->def->history;
c->curprg->def->history = 0;
 
diff --git a/sql/test/malloc_fail/Tests/All b/sql/test/malloc_fail/Tests/All
--- a/sql/test/malloc_fail/Tests/All
+++ b/sql/test/malloc_fail/Tests/All
@@ -1,1 +1,1 @@
-# mallocs not run for now since it hangs mserver
+# mallocs still hangs
diff --git a/sql/test/malloc_fail/Tests/mallocs.SQL.py 
b/sql/test/malloc_fail/Tests/mallocs.SQL.py
--- a/sql/test/malloc_fail/Tests/mallocs.SQL.py
+++ b/sql/test/malloc_fail/Tests/mallocs.SQL.py
@@ -1,6 +1,10 @@
 import pymonetdb
 import os
 
+import logging
+logging.basicConfig()
+
+
 dbh = pymonetdb.connect(database = os.environ['TSTDB'],
 port = int(os.environ['MAPIPORT']),
 hostname = "localhost")
@@ -15,14 +19,15 @@ create procedure setmallocsuccesscount(c
 call setmallocsuccesscount(%d);
 SELECT * FROM tables;
 """
-i = 3300
+i = 3000
+
 while i > 1000:
-#print(i)
+print(i)
 i-=1
 try:
 cursor.execute(q % (i))
 except Exception as e:
-#print(e)
+  # print(e)
 pass
 finally:
 dbh.rollback()
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - more unchecked malloc, this time in optimizer...

2017-03-10 Thread Hannes Muehleisen

Changeset: e18c02b1ad0e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e18c02b1ad0e
Modified Files:
monetdb5/optimizer/opt_dataflow.c
monetdb5/optimizer/opt_support.c
monetdb5/optimizer/opt_wrapper.c
sql/backends/monet5/sql_execute.c
Branch: default
Log Message:

more unchecked malloc, this time in optimizers. Changed contract for optimizer 
function return code, values < 0 signal error and abort query.


diffs (86 lines):

diff --git a/monetdb5/optimizer/opt_dataflow.c 
b/monetdb5/optimizer/opt_dataflow.c
--- a/monetdb5/optimizer/opt_dataflow.c
+++ b/monetdb5/optimizer/opt_dataflow.c
@@ -201,9 +201,10 @@ OPTdataflowImplementation(Client cntxt, 
limit= mb->stop;
slimit= mb->ssize;
old = mb->stmt;
-   if ( newMalBlkStmt(mb, mb->ssize) <0 )
+   if (newMalBlkStmt(mb, mb->ssize) < 0) {
+   actions = -1;
goto wrapup;
-   
+   }
pushInstruction(mb,old[0]);
 
/* inject new dataflow barriers using a single pass through the program 
*/
@@ -320,9 +321,9 @@ OPTdataflowImplementation(Client cntxt, 
 newComment(mb,buf);
 
 wrapup:
-   if( states) GDKfree(states);
-   if( sink) GDKfree(sink);
-   if( old) GDKfree(old);
+   if(states) GDKfree(states);
+   if(sink)   GDKfree(sink);
+   if(old)GDKfree(old);
 
return actions;
 }
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -125,7 +125,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
/* force at least once a complete type check by resetting the type 
check flag */
 
resetMalBlk(mb, mb->stop);
-   chkProgram(cntxt->fdout, cntxt->nspace,mb);
+   chkProgram(cntxt->fdout, cntxt->nspace, mb);
if (mb->errors)
throw(MAL, "optimizer.MALoptimizer", "Start with inconsistent 
MAL plan");
 
diff --git a/monetdb5/optimizer/opt_wrapper.c b/monetdb5/optimizer/opt_wrapper.c
--- a/monetdb5/optimizer/opt_wrapper.c
+++ b/monetdb5/optimizer/opt_wrapper.c
@@ -143,12 +143,15 @@ str OPTwrapper (Client cntxt, MalBlkPtr 
} else if( p ) 
removeInstruction(mb, p);
 
-   for ( i=0; codes[i].nme; i++)
-   if ( strcmp(codes[i].nme, optimizer)== 0 ){
-   actions = (int)(*(codes[i].fcn))(cntxt, mb, stk,0);
+   for (i=0; codes[i].nme; i++)
+   if (strcmp(codes[i].nme, optimizer) == 0){
+   actions = (int)(*(codes[i].fcn))(cntxt, mb, stk, 0);
+   if (actions < 1) {
+   throw(MAL, optimizer, "Error in optimizer %s", 
optimizer);
+   }
break;  
}
-   if ( codes[i].nme == 0)
+   if (codes[i].nme == 0)
throw(MAL, optimizer, "Optimizer implementation '%s' missing", 
fcnnme);
 
OPTIMIZERDEBUG {
diff --git a/sql/backends/monet5/sql_execute.c 
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -276,7 +276,7 @@ SQLrun(Client c, backend *be, mvc *m){
 
/* only consider a re-optimization when we are dealing with query 
templates */
for ( i= 1; i < mb->stop;i++){
-   p=getInstrPtr(mb,i);
+   p = getInstrPtr(mb,i);
if( getFunctionId(p) &&  
qc_isapreparedquerytemplate(getFunctionId(p) ) ){
msg = SQLexecutePrepared(c, be, p->blk);
freeMalBlk(mb);
@@ -284,6 +284,9 @@ SQLrun(Client c, backend *be, mvc *m){
}
if( getFunctionId(p) &&  p->blk && 
qc_isaquerytemplate(getFunctionId(p)) ) {
mc = copyMalBlk(p->blk);
+   if (!mc) {
+   throw(SQL, "sql.prepare", "Out of memory");
+   }
retc = p->retc;
freeMalBlk(mb);
mb = mc;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - unchecked malloc, found with mallocs.SQL.py

2017-03-10 Thread Hannes Muehleisen

Changeset: ebff12da64d1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ebff12da64d1
Modified Files:
monetdb5/mal/mal_instruction.c
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_support.c
sql/backends/monet5/sql_execute.c
Branch: default
Log Message:

unchecked malloc, found with mallocs.SQL.py


diffs (74 lines):

diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -392,10 +392,9 @@ newInstruction(MalBlkPtr mb, str modnme,
 InstrPtr
 copyInstruction(InstrPtr p)
 {
-   InstrPtr new;
-   new = (InstrPtr) GDKmalloc(offsetof(InstrRecord, argv) + p->maxarg * 
sizeof(p->maxarg));
-   if( new == NULL) {
-   GDKerror("copyInstruction:failed to allocated space");
+   InstrPtr new = (InstrPtr) GDKmalloc(offsetof(InstrRecord, argv) + 
p->maxarg * sizeof(p->maxarg));
+   if(new == NULL) {
+   GDKerror("copyInstruction: failed to allocated space");
return new;
}
oldmoveInstruction(new, p);
diff --git a/monetdb5/optimizer/opt_pipes.c b/monetdb5/optimizer/opt_pipes.c
--- a/monetdb5/optimizer/opt_pipes.c
+++ b/monetdb5/optimizer/opt_pipes.c
@@ -495,6 +495,9 @@ addOptimizerPipe(Client cntxt, MalBlkPtr
if (pipes[i].mb) {
for (j = 1; j < pipes[i].mb->stop - 1; j++) {
p = copyInstruction(pipes[i].mb->stmt[j]);
+   if (!p) { // oh malloc you cruel mistress
+   throw(MAL, "optimizer.addOptimizerPipe", "Out 
of memory");
+   }
for (k = 0; k < p->argc; k++)
getArg(p, k) = cloneVariable(mb, pipes[i].mb, 
getArg(p, k));
typeChecker(cntxt->fdout, cntxt->nspace, mb, p, FALSE);
diff --git a/monetdb5/optimizer/opt_support.c b/monetdb5/optimizer/opt_support.c
--- a/monetdb5/optimizer/opt_support.c
+++ b/monetdb5/optimizer/opt_support.c
@@ -124,8 +124,8 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
 
/* force at least once a complete type check by resetting the type 
check flag */
 
-   resetMalBlk(mb,mb->stop);
-   chkProgram(cntxt->fdout,cntxt->nspace,mb);
+   resetMalBlk(mb, mb->stop);
+   chkProgram(cntxt->fdout, cntxt->nspace,mb);
if (mb->errors)
throw(MAL, "optimizer.MALoptimizer", "Start with inconsistent 
MAL plan");
 
@@ -133,7 +133,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
  * When no optimzer call is found, be terminate. */
do {
qot = 0;
-   for (pc = 0; pc < mb->stop ; pc++) {
+   for (pc = 0; pc < mb->stop; pc++) {
p = getInstrPtr(mb, pc);
if (getModuleId(p) == optimizerRef && p->fcn && 
p->token != REMsymbol) {
/* all optimizers should behave like patterns */
@@ -143,7 +143,7 @@ optimizeMALBlock(Client cntxt, MalBlkPtr
msg = (str) (*p->fcn) (cntxt, mb, 0, p);
if (msg) {
str place = getExceptionPlace(msg);
-   str nmsg= 
createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg));
+   str nmsg = 
createException(getExceptionType(msg), place, "%s", getExceptionMessage(msg));
GDKfree(place);
GDKfree(msg);
msg = nmsg;
diff --git a/sql/backends/monet5/sql_execute.c 
b/sql/backends/monet5/sql_execute.c
--- a/sql/backends/monet5/sql_execute.c
+++ b/sql/backends/monet5/sql_execute.c
@@ -310,7 +310,7 @@ SQLrun(Client c, backend *be, mvc *m){
// This include template constants, BAT sizes.
if( m->emod & mod_debug)
mb->keephistory = TRUE;
-   msg = SQLoptimizeQuery(c,mb);
+   msg = SQLoptimizeQuery(c, mb);
mb->keephistory = FALSE;
 
if( mb->errors){
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - forcing dplyr dev

2017-03-06 Thread Hannes Muehleisen

Changeset: 8e3dccb2c60f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8e3dccb2c60f
Modified Files:
clients/R/Tests/deps-test.R
Branch: Dec2016
Log Message:

forcing dplyr dev


diffs (16 lines):

diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R
--- a/clients/R/Tests/deps-test.R
+++ b/clients/R/Tests/deps-test.R
@@ -5,4 +5,12 @@ dd <- capture.output(suppressMessages(su
if(length(np)) install.packages(np, repos=repos, quiet=T)
update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T)
})(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata', 'survey'))
+
+   # dev dplyr
+   if (packageVersion("devtools") < 1.6) {
+ install.packages("devtools")
+   }
+   devtools::install_github("hadley/lazyeval")
+   devtools::install_github("hadley/dplyr")
+
 })))
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - added survey dependency

2017-03-06 Thread Hannes Muehleisen

Changeset: 528fecb20eee for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=528fecb20eee
Modified Files:
clients/R/Tests/deps-test.R
Branch: Dec2016
Log Message:

added survey dependency


diffs (10 lines):

diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R
--- a/clients/R/Tests/deps-test.R
+++ b/clients/R/Tests/deps-test.R
@@ -4,5 +4,5 @@ dd <- capture.output(suppressMessages(su
repos <- 'http://cran.rstudio.com/'
if(length(np)) install.packages(np, repos=repos, quiet=T)
update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T)
-   })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata'))
+   })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata', 'survey'))
 })))
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - finally fixing R test cases

2017-03-03 Thread Hannes Muehleisen

Changeset: 65191bb72656 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=65191bb72656
Modified Files:
clients/R/Tests/copy_into_fwf.stable.out
clients/R/Tests/dbapply.stable.out
clients/R/Tests/dbi.R
clients/R/Tests/dbi.stable.err
clients/R/Tests/dbi.stable.out
clients/R/Tests/dplyr-flights.R
clients/R/Tests/dplyr-flights.stable.out
clients/R/Tests/dplyr.R
clients/R/Tests/dplyr.stable.err
clients/R/Tests/dplyr.stable.out
Branch: Dec2016
Log Message:

finally fixing R test cases


diffs (truncated from 361 to 300 lines):

diff --git a/clients/R/Tests/copy_into_fwf.stable.out 
b/clients/R/Tests/copy_into_fwf.stable.out
--- a/clients/R/Tests/copy_into_fwf.stable.out
+++ b/clients/R/Tests/copy_into_fwf.stable.out
@@ -24,11 +24,8 @@ Ready.
 # 16:40:23 >  "R" "--vanilla" "--slave" "--args" "30658"
 # 16:40:23 >  
 
-[1] TRUE
 
   SQL  CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp 
DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE 
PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, 
gear DOUBLE PRECISION, carb DOUBLE PRECISION)
-  ROWS Fetched: 0 [complete]
-   Changed: NA
 mpg cyl  disp  hp dratwt  qsec vs am gear carb
 1  21.0   6 160.0 110 3.90 2.620 16.46  0  144
 2  21.0   6 160.0 110 3.90 2.875 17.02  0  144
@@ -62,7 +59,6 @@ 29 15.8   8 351.0 264 4.22 3.170 14.50  
 30 19.7   6 145.0 175 3.62 2.770 15.50  0  156
 31 15.0   8 301.0 335 3.54 3.570 14.60  0  158
 32 21.4   4 121.0 109 4.11 2.780 18.60  1  142
-[1] TRUE
 [1] "SUCCESS"
 
 # 16:40:23 >  
diff --git a/clients/R/Tests/dbapply.stable.out 
b/clients/R/Tests/dbapply.stable.out
--- a/clients/R/Tests/dbapply.stable.out
+++ b/clients/R/Tests/dbapply.stable.out
@@ -29,14 +29,10 @@ Ready.
 # 11:43:14 >  "R" "--vanilla" "--slave" "--args" "35780"
 # 11:43:14 >  
 
-[1] TRUE
-[1] TRUE
-[1] TRUE
 [1] 32
 [1] TRUE
 [1] 32
 [1] 42
-[1] TRUE
 [1] "SUCCESS"
 
 # 11:43:21 >  
diff --git a/clients/R/Tests/dbi.R b/clients/R/Tests/dbi.R
--- a/clients/R/Tests/dbi.R
+++ b/clients/R/Tests/dbi.R
@@ -43,6 +43,7 @@ dbRemoveTable(con,tname)
 stopifnot(identical(dbExistsTable(con,tname),FALSE))
 
 # write test table iris
+# iris is one of the built-in datasets in R
 data(iris)
 dbWriteTable(con,tname,iris)
 
@@ -65,16 +66,22 @@ stopifnot(identical(res@env$success,TRUE
 stopifnot(dbColumnInfo(res)[[1,1]] == "Species")
 stopifnot(dbColumnInfo(res)[[2,1]] == "Sepal.Width")
 
-stopifnot(dbGetRowCount(res) == 150 && res@env$info$rows == 150)
+stopifnot(dbGetRowCount(res) == 0)
 
 data <- dbFetch(res,10)
 
+stopifnot(dbGetRowCount(res) == 10)
+
+
 stopifnot(dim(data)[[1]] == 10)
 stopifnot(dim(data)[[2]] == 2)
 stopifnot(res@env$delivered == 10)
 stopifnot(dbHasCompleted(res) == FALSE)
 
 data2 <- dbFetch(res,-1)
+
+stopifnot(dbGetRowCount(res) == 150)
+
 stopifnot(dim(data2)[[1]] == 140)
 stopifnot(dbHasCompleted(res) == TRUE)
 
diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err
--- a/clients/R/Tests/dbi.stable.err
+++ b/clients/R/Tests/dbi.stable.err
@@ -37,6 +37,7 @@ Use 'dbIsValid' instead.
 See help("Deprecated") 
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
+Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Treating character vector parameter as file name(s) for monetdb.read.csv()
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
diff --git a/clients/R/Tests/dbi.stable.out b/clients/R/Tests/dbi.stable.out
--- a/clients/R/Tests/dbi.stable.out
+++ b/clients/R/Tests/dbi.stable.out
@@ -44,42 +44,12 @@ Ready.
 # loading sql script: 90_generator.sql
 # loading sql script: 99_system.sql
 
-# 12:26:31 >  
-# 12:26:31 >  "R" "--vanilla" "--slave" "--args" "31372"
-# 12:26:31 >  
+# 15:00:13 >  
+# 15:00:13 >  "R" "--vanilla" "--slave" "--args" "34632"
+# 15:00:13 >  
 
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
 [1] "\"Need to quote this table name\"" "monetdbtest"  
 [3] "othermethod"  
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
-[1] TRUE
 [1] "SUCCESS"
 
 # 12:26:33 >  
diff

MonetDB: queryid2 - closing branch

2017-02-17 Thread Hannes Muehleisen

Changeset: 53e82e18e17b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=53e82e18e17b
Branch: queryid2
Log Message:

closing branch

___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - merged in queryid2

2017-02-17 Thread Hannes Muehleisen

Changeset: 1a3bd9a93d80 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1a3bd9a93d80
Modified Files:
monetdb5/mal/mal_runtime.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
sql/include/sql_catalog.h
sql/storage/bat/res_table.c
sql/storage/sql_storage.h

sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out

sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out

sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out
sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py
sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out
sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out
sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out
sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out.int128
sql/test/Tests/identifiers.stable.out
sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py
sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out
sql/test/VOC/Tests/median.Bug-3096.stable.out
sql/test/bugs/Tests/aggr_single_value.stable.out
sql/test/bugs/Tests/aggr_single_value.stable.out.int128
sql/test/pg_regress/Tests/date.stable.out
sql/test/pg_regress/Tests/interval.stable.out
sql/test/pg_regress/Tests/interval.stable.out.int128
sql/test/pg_regress/Tests/strings.stable.out
sql/test/pg_regress/Tests/strings_concat.stable.out
Branch: default
Log Message:

merged in queryid2


diffs (truncated from 1014 to 300 lines):

diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c
--- a/monetdb5/mal/mal_runtime.c
+++ b/monetdb5/mal/mal_runtime.c
@@ -83,6 +83,7 @@ runtimeProfileInit(Client cntxt, MalBlkP
if (i == qtop) {
QRYqueue[i].mb = mb;
QRYqueue[i].tag = qtag++;
+   mb->tag = QRYqueue[i].tag;
QRYqueue[i].stk = stk;  // for status 
pause 'p'/running '0'/ quiting 'q'
QRYqueue[i].start = (lng)time(0);
QRYqueue[i].runtime = mb->runtime;  // the estimated 
execution time
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -1705,7 +1705,7 @@ mvc_result_set_wrap( Client cntxt, MalBl
b = BATdescriptor(bid);
if ( b == NULL)
throw(MAL,"sql.resultset","Failed to access order column");
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, b);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
5), 1, b);
if (res < 0)
msg = createException(SQL, "sql.resultSet", "failed");
BBPunfix(b->batCacheid);
@@ -1796,7 +1796,7 @@ mvc_export_table_wrap( Client cntxt, Mal
order = BATdescriptor(bid);
if ( order == NULL)
throw(MAL,"sql.resultset","Failed to access order column");
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, 
order);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
11), 1, order);
t = m->results;
if (res < 0){
msg = createException(SQL, "sql.resultSet", "failed");
@@ -1895,7 +1895,7 @@ mvc_row_result_wrap( Client cntxt, MalBl
return msg;
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, 
NULL);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
5), 1, NULL);
 
tbl = BATdescriptor(tblId);
atr = BATdescriptor(atrId);
@@ -1970,7 +1970,7 @@ mvc_export_row_wrap( Client cntxt, MalBl
return msg;
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, 
NULL);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
11), 1, NULL);
 
t = m->results;
if (res < 0){
@@ -2071,7 +2071,7 @@ mvc_table_result_wrap(Client cntxt, MalB
if ((order = BATdescriptor(*order_bid)) == NULL) {
throw(SQL, "sql.resultSet", "Cannot access descriptor");
}
-   *res_id = mvc_result_table(m, *nr_cols, *qtype, order);
+   *res_id = mvc_result_table(m, mb->tag, *nr_cols, *qtype, order);
if (*res_id < 0)
res = createException(SQL, "sql.resultSet", "failed");

MonetDB: queryid2 - merge with default

2017-02-17 Thread Hannes Muehleisen

Changeset: b14c4bcbf142 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b14c4bcbf142
Added Files:
sql/test/BugTracker-2016/Tests/storagemodel.stable.out.Windows
Removed Files:
debian/libmonetdb5-server-gsl.install
sql/backends/monet5/gsl/46_gsl.sql
sql/backends/monet5/gsl/73_gsl.mal
sql/backends/monet5/gsl/Makefile.ag
sql/backends/monet5/gsl/Tests/All
sql/backends/monet5/gsl/Tests/gsl00.malC
sql/backends/monet5/gsl/Tests/gsl00.stable.err
sql/backends/monet5/gsl/Tests/gsl00.stable.out
sql/backends/monet5/gsl/Tests/gsl01.sql
sql/backends/monet5/gsl/Tests/gsl01.stable.err
sql/backends/monet5/gsl/Tests/gsl01.stable.out
sql/backends/monet5/gsl/gsl.c
sql/backends/monet5/gsl/gsl.mal
sql/test/BugTracker-2016/Tests/storagemodel.stable.out.32bit
sql/test/BugTracker-2016/Tests/storagemodel.stable.out.int128
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
clients/Tests/All
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
configure.ag
debian/changelog
debian/control
debian/rules
gdk/libbat.rc
libversions
monetdb5/mal/mal_resource.h
monetdb5/tools/libmonetdb5.rc
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql.c
sql/test/BugTracker-2013/Tests/All
sql/test/BugTracker-2015/Tests/All
sql/test/BugTracker-2016/Tests/storagemodel.sql
sql/test/BugTracker-2016/Tests/storagemodel.stable.err
sql/test/BugTracker-2016/Tests/storagemodel.stable.out
sql/test/Tests/All
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/All
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/All
sql/test/emptydb-upgrade-chain/Tests/check.stable.out
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/All
sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade/Tests/All
sql/test/emptydb-upgrade/Tests/check.stable.out
sql/test/emptydb-upgrade/Tests/check.stable.out.32bit
sql/test/emptydb-upgrade/Tests/check.stable.out.int128
sql/test/emptydb/Tests/All
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.32bit
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/leaks/Tests/All
sql/test/leaks/Tests/check1.stable.out
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.stable.out
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.stable.out
sql/test/leaks/Tests/check3.stable.out.int128
sql/test/leaks/Tests/check4.stable.out
sql/test/leaks/Tests/check4.stable.out.int128
sql/test/leaks/Tests/check5.stable.out
sql/test/leaks/Tests/check5.stable.out.int128
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.out
sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.sql

sql/test/sys-schema/Tests/check_ForeignKey_referential_integrity.stable.out
sql/test/sys-schema/Tests/check_PrimaryKey_uniqueness.sql
sql/test/sys-schema/Tests/check_PrimaryKey_uniqueness.stable.out
testing/Mtest.py.in
vertoo.data
Branch: queryid2
Log Message:

merge with default


diffs (truncated from 3384 to 300 lines):

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -649,3 +649,5 @@ ea0fe76e0095771bccca626b39e3e839a90ba320
 afc859894553ef804cded49a83b035d9d2cda73b Dec2016_release
 bf4b58d202645774b75c7f508c84b30e3be78a49 Dec2016_5
 bf4b58d202645774b75c7f508c84b30e3be78a49 Dec2016_SP1_release
+3968ed7fb6dab04a2530c6c3e1a5ffd6d649298b Dec2016_7
+3968ed7fb6dab04a2530c6c3e1a5ffd6d649298b Dec2016_SP2_release
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -78,6 +78,14 @@
 %endif
 %endif
 
+# If the _without_pcre macro is not set, the PCRE library is used for
+# the implementation of the SQL LIKE and ILIKE operators.  Otherwise
+# the

MonetDB: noconsole - startup messages don't have to start with #

2017-02-17 Thread Hannes Muehleisen

Changeset: 99bcc306c7aa for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=99bcc306c7aa
Modified Files:
geom/lib/libgeom.c
monetdb5/extras/rapi/rapi.c
monetdb5/mal/mal_profiler.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/UDF/pyapi/pyapi.c
sql/backends/monet5/sql_scenario.c
tools/mserver/mserver5.c
Branch: noconsole
Log Message:

startup messages don't have to start with #


diffs (139 lines):

diff --git a/geom/lib/libgeom.c b/geom/lib/libgeom.c
--- a/geom/lib/libgeom.c
+++ b/geom/lib/libgeom.c
@@ -23,7 +23,7 @@ libgeom_init(void)
 {
initGEOS((GEOSMessageHandler) GDKerror, (GEOSMessageHandler) GDKerror);
GEOS_setWKBByteOrder(1);/* NDR (little endian) */
-   printf("# MonetDB/GIS module loaded\n");
+   printf("MonetDB/GIS module loaded\n");
fflush(stdout); /* make merovingian see this *now* */
 }
 
diff --git a/monetdb5/extras/rapi/rapi.c b/monetdb5/extras/rapi/rapi.c
--- a/monetdb5/extras/rapi/rapi.c
+++ b/monetdb5/extras/rapi/rapi.c
@@ -511,7 +511,7 @@ str RAPIprelude(void *ret) {
 
}
MT_lock_unset();
-   printf("# MonetDB/R   module loaded\n");
+   printf("MonetDB/R   module loaded\n");
}
return MAL_SUCCEED;
 }
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -1023,7 +1023,7 @@ void setHeartbeat(int delay)
 void initProfiler(void)
 {
if( mal_trace)
-   openProfilerStream(mal_clients[0].fdout,0);
+   openProfilerStream(GDKout, 0);
 }
 
 void initHeartbeat(void)
diff --git a/monetdb5/modules/mal/clients.c b/monetdb5/modules/mal/clients.c
--- a/monetdb5/modules/mal/clients.c
+++ b/monetdb5/modules/mal/clients.c
@@ -613,7 +613,7 @@ CLTsessions(Client cntxt, MalBlkPtr mb, 

 MT_lock_set(_contextLock);

-for (c = mal_clients + (GDKgetenv_isyes("monet_daemon") != 0); c < 
mal_clients + MAL_MAXCLIENTS; c++) 
+for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++)
if (c->mode == RUNCLIENT) {
BUNappend(user, c->username, FALSE);
msg = MTIMEunix_epoch();
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -433,7 +433,7 @@ static void SERVERannounce(struct in_add
free(buf);
else
/* announce that we're now reachable */
-   printf("# Listening for connection requests on "
+   printf("Listening for connection requests on "
"mapi:monetdb://%s:%i/\n", host, port);
}
if (usockfile != NULL) {
@@ -442,7 +442,7 @@ static void SERVERannounce(struct in_add
free(buf);
else
/* announce that we're now reachable */
-   printf("# Listening for UNIX domain connection requests 
on "
+   printf("Listening for UNIX domain connection requests 
on "
"mapi:monetdb://%s\n", usockfile);
}
 }
diff --git a/sql/backends/monet5/UDF/pyapi/pyapi.c 
b/sql/backends/monet5/UDF/pyapi/pyapi.c
--- a/sql/backends/monet5/UDF/pyapi/pyapi.c
+++ b/sql/backends/monet5/UDF/pyapi/pyapi.c
@@ -1288,7 +1288,7 @@ PyAPIprelude(void *ret) {
 pyapiInitialized++;
 }
 MT_lock_unset();
-fprintf(stdout, "# MonetDB/Python module loaded\n");
+fprintf(stdout, "MonetDB/Python module loaded\n");
 option_disable_fork = GDKgetenv_istrue(fork_disableflag) || 
GDKgetenv_isyes(fork_disableflag);
 return MAL_SUCCEED;
 }
diff --git a/sql/backends/monet5/sql_scenario.c 
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -169,7 +169,7 @@ SQLprelude(void *ret)
if (tmp != MAL_SUCCEED)
return (tmp);
 #ifndef HAVE_EMBEDDED
-   fprintf(stdout, "# MonetDB/SQL module loaded\n");
+   fprintf(stdout, "MonetDB/SQL module loaded\n");
fflush(stdout); /* make merovingian see this *now* */
 #endif
/* only register availability of scenarios AFTER we are inited! */
diff --git a/tools/mserver/mserver5.c b/tools/mserver/mserver5.c
--- a/tools/mserver/mserver5.c
+++ b/tools/mserver/mserver5.c
@@ -139,16 +139,16 @@ monet_hello(void)
qi++;
}
 
-   printf("# MonetDB 5 server v" VERSION);
+   printf("MonetDB 5 server v" VERSION);
/* coverity[pointless_string_compare] */
if (strcmp(MONETDB_RELEASE, "unreleased") == 0)
-   printf("\n# This is an unreleased version");
+   printf("\nThis is an unreleased version");
else

MonetDB: noconsole - more server console special cases removed

2017-02-17 Thread Hannes Muehleisen

Changeset: 006d512ba9b0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=006d512ba9b0
Modified Files:
monetdb5/mal/mal.c
monetdb5/mal/mal_authorize.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_scenario.c
monetdb5/mal/mal_session.c
monetdb5/modules/mal/clients.c
monetdb5/modules/mal/mal_mapi.c
Branch: noconsole
Log Message:

more server console special cases removed


diffs (264 lines):

diff --git a/monetdb5/mal/mal.c b/monetdb5/mal/mal.c
--- a/monetdb5/mal/mal.c
+++ b/monetdb5/mal/mal.c
@@ -144,14 +144,6 @@ void mserver_reset(int exit)
 */
mal_factory_reset();
mal_dataflow_reset();
-   THRdel(mal_clients->mythread);
-   GDKfree(mal_clients->errbuf);
-   mal_clients->fdin->s = NULL;
-   bstream_destroy(mal_clients->fdin);
-   GDKfree(mal_clients->prompt);
-   GDKfree(mal_clients->username);
-   freeStack(mal_clients->glb);
-   freeSymbol(mal_clients->curprg);
mal_client_reset();
mal_linker_reset();
mal_resource_reset();
diff --git a/monetdb5/mal/mal_authorize.c b/monetdb5/mal/mal_authorize.c
--- a/monetdb5/mal/mal_authorize.c
+++ b/monetdb5/mal/mal_authorize.c
@@ -229,7 +229,7 @@ AUTHinitTables(const char *passwd) {
char *pw;
oid uid;
Client c = _clients[0];
-
+   // TODO: does this still work without the server console?
if (passwd == NULL)
passwd = "monetdb"; /* default password */
pw = mcrypt_BackendSum(passwd, strlen(passwd));
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -442,7 +442,7 @@ MCactiveClients(void)
int freeclient=0, finishing=0, running=0, blocked = 0;
Client cntxt = mal_clients;
 
-   for(cntxt= mal_clients;  cntxtmode == FREECLIENT);
finishing += (cntxt->mode == FINISHCLIENT);
running += (cntxt->mode == RUNCLIENT);
diff --git a/monetdb5/mal/mal_debugger.c b/monetdb5/mal/mal_debugger.c
--- a/monetdb5/mal/mal_debugger.c
+++ b/monetdb5/mal/mal_debugger.c
@@ -388,18 +388,13 @@ mdbCommand(Client cntxt, MalBlkPtr mb, M
char oldcmd[1024] = { 0 };
do {
if (p != NULL) {
-   if (cntxt != mal_clients)
-   /* help mclients with fake prompt */
-   if (lastcmd != 'l' && lastcmd != 'L') {
-   mnstr_printf(out, "mdb>");
-   printTraceCall(out, mb, stk, pc, 
LIST_MAL_CALL);
-   }
+   /* help mclients with fake prompt */
+   if (lastcmd != 'l' && lastcmd != 'L') {
+   mnstr_printf(out, "mdb>");
+   printTraceCall(out, mb, stk, pc, LIST_MAL_CALL);
+   }
+   }
 
-   }
-   if (cntxt == mal_clients) {
-   cntxt->prompt = "mdb>";
-   cntxt->promptlength = 4;
-   }
 
if (cntxt->phase[MAL_SCENARIO_READER]) {
 retryRead:
@@ -414,13 +409,7 @@ retryRead:
goto retryRead;
}
}
-#ifndef HAVE_EMBEDDED
-   else if (cntxt == mal_clients) {
-   /* switch to mdb streams */
-   if (readConsole(cntxt) <= 0)
-   break;
-   }
-#endif
+
b = CURRENT(cntxt);
 
/* terminate the line with zero */
@@ -974,13 +963,13 @@ str mdbTrap(Client cntxt, MalBlkPtr mb, 
int cnt = 20;   /* total 10 sec delay */
int pc = getPC(mb,p);
 
-   mnstr_printf(mal_clients[0].fdout, "#trapped %s.%s[%d]\n",
+   mnstr_printf(cntxt->fdout, "#trapped %s.%s[%d]\n",
getModuleId(mb->stmt[0]), getFunctionId(mb->stmt[0]), 
pc);
-   printInstruction(mal_clients[0].fdout, mb, stk, p, LIST_MAL_DEBUG);
+   printInstruction(cntxt->fdout, mb, stk, p, LIST_MAL_DEBUG);
cntxt->itrace = 'W';
MT_lock_set(_contextLock);
if (trapped_mb) {
-   mnstr_printf(mal_clients[0].fdout, "#registry not available\n");
+   mnstr_printf(cntxt->fdout, "#registry not available\n");
mnstr_flush(cntxt->fdout);
}
while (trapped_mb && cnt-- > 0) {
@@ -1015,12 +1004,12 @@ mdbStep(Client cntxt, MalBlkPtr mb, MalS
state.p = getInstrPtr(mb, pc);
state.pc = pc;
cntxt->mdb = 
-

MonetDB: noconsole - console gone from implementation, mtest sti...

2017-02-17 Thread Hannes Muehleisen

Changeset: 8ec2f8a7977b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8ec2f8a7977b
Modified Files:
configure.ag
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_session.c
sql/backends/monet5/mal_backend.c
sql/backends/monet5/mal_backend.h
sql/backends/monet5/sql_scenario.c
tools/mserver/mserver5.c
Branch: noconsole
Log Message:

console gone from implementation, mtest still needs a fix


diffs (248 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -292,20 +292,6 @@ AC_ARG_ENABLE([pyintegration],
[enable_pyintegration=$enableval],
[enable_pyintegration=$dft_pyintegration])
 
-# The console is a direct client hooked onto the kernel with full
-# administrative privileges, bypassing any security checks.  It is
-# handy only during development.
-# We simply can't disable it for now because Testing gets hopelessly
-# upset about it, but we can offer to the user to disable it for
-# increased security.
-dft_console=yes
-AC_ARG_ENABLE([console],
-   [AS_HELP_STRING([--enable-console],
-   [enables direct console on the server (involves security risks) 
(default=yes)])],
-   [enable_console="$enableval"],
-   [enable_console=$dft_console])
-AS_VAR_IF([enable_console], [no], [],
-   [AC_DEFINE([HAVE_CONSOLE], 1, [If the console should be used])])
 
 dft_int128=auto
 AC_ARG_ENABLE([int128],
@@ -3049,7 +3035,6 @@ for comp in \
'monetdb5' \
'sql ' \
'geom' \
-   'console ' \
'pyintegration' \
'rintegration' \
'odbc' \
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -127,11 +127,7 @@ MCnewClient(void)
 {
Client c;
MT_lock_set(_contextLock);
-   if (mal_clients[CONSOLE].user && mal_clients[CONSOLE].mode == 
FINISHCLIENT) {
-   /*system shutdown in progress */
-   MT_lock_unset(_contextLock);
-   return NULL;
-   }
+   // TODO: check some exiting flag here?
for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++) {
if (c->mode == FREECLIENT) {
c->mode = RUNCLIENT;
@@ -429,7 +425,7 @@ MCstopClients(Client cntxt)
Client c = mal_clients;
 
MT_lock_set(_contextLock);
-   for(c= mal_clients +1;  c < mal_clients+MAL_MAXCLIENTS; c++)
+   for(c= mal_clients ;  c < mal_clients+MAL_MAXCLIENTS; c++)
if( cntxt != c){
if ( c->mode == RUNCLIENT)
c->mode = FINISHCLIENT; 
@@ -446,7 +442,7 @@ MCactiveClients(void)
int freeclient=0, finishing=0, running=0, blocked = 0;
Client cntxt = mal_clients;
 
-   for(cntxt= mal_clients+1;  cntxtmode == FREECLIENT);
finishing += (cntxt->mode == FINISHCLIENT);
running += (cntxt->mode == RUNCLIENT);
@@ -462,14 +458,8 @@ MCcloseClient(Client c)
printf("closeClient %d " OIDFMT "\n", (int) (c - mal_clients), c->user);
 #endif
/* free resources of a single thread */
-   if (!isAdministrator(c)) {
-   freeClient(c);
-   return;
-   }
+   freeClient(c);
 
-   /* adm is set to disallow new clients entering */
-   mal_clients[CONSOLE].mode = FINISHCLIENT;
-   mal_exit();
 }
 
 str
diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h
--- a/monetdb5/mal/mal_client.h
+++ b/monetdb5/mal/mal_client.h
@@ -15,9 +15,6 @@
 #include "mal_resolve.h"
 #include "mal_profiler.h"
 
-#define CONSOLE 0
-#define isAdministrator(X) (X==mal_clients)
-
 enum clientmode {
FREECLIENT,
FINISHCLIENT,
diff --git a/monetdb5/mal/mal_session.c b/monetdb5/mal/mal_session.c
--- a/monetdb5/mal/mal_session.c
+++ b/monetdb5/mal/mal_session.c
@@ -425,7 +425,7 @@ MSserveClient(void *dummy)
Client c = (Client) dummy;
str msg = 0;
 
-   if (!isAdministrator(c) && MCinitClientThread(c) < 0) {
+   if (MCinitClientThread(c) < 0) {
MCcloseClient(c);
return;
}
@@ -482,15 +482,8 @@ MSserveClient(void *dummy)
assert(0);
}
 
-   if (c->mode > FINISHCLIENT) {
-   if (isAdministrator(c) /* && moreClients(0)==0 */) {
-   if (c->scenario) {
-   exitScenario(c);
-   }
-   }
-   }
-   if (!isAdministrator(c))
-   MCcloseClient(c);
+   MCcloseClient(c);
+
if (c->nspace && strcmp(c->nspace->name, "user") == 0) {
GDKfree(c->nspace->space);
GDKfree(c->nspace);
diff --git

MonetDB: queryid2 - more test output approved

2017-02-16 Thread Hannes Muehleisen

Changeset: 3a0b19bb5587 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3a0b19bb5587
Modified Files:
sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py
sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out
sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out
sql/test/BugTracker/Tests/case_with_aggr.SF-1876779.stable.out.int128
sql/test/bugs/Tests/aggr_single_value.stable.out
sql/test/bugs/Tests/aggr_single_value.stable.out.int128
sql/test/pg_regress/Tests/interval.stable.out
sql/test/pg_regress/Tests/interval.stable.out.int128
Branch: queryid2
Log Message:

more test output approved


diffs (truncated from 303 to 300 lines):

diff --git a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py 
b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py
--- a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py
+++ b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.SQL.py
@@ -1,10 +1,28 @@
 import os
-from pymonetdb import mapi
+import pymonetdb
 
-def open(conn):
-
conn.connect(database=os.getenv('TSTDB'),username="monetdb",password="monetdb",language="sql",hostname=os.getenv('MAPIHOST'),port=int(os.getenv('MAPIPORT')))
+def connect():
+return pymonetdb.connect(database = os.getenv('TSTDB'),
+ hostname = '127.0.0.1',
+ port = int(os.getenv('MAPIPORT')),
+ username = 'monetdb',
+ password = 'monetdb',
+ autocommit = True)
 
-works = """-- some comment
+def query(conn, sql, result=False):
+cur = conn.cursor()
+cur.execute(sql)
+r = False
+if (result):
+r = cur.fetchall()
+cur.close()
+return r
+
+
+# def open(conn):
+# 
conn.connect(database=os.getenv('TSTDB'),username="monetdb",password="monetdb",language="sql",hostname=os.getenv('MAPIHOST'),port=int(os.getenv('MAPIPORT')))
+
+create1 = """-- some comment
 START TRANSACTION;
 CREATE TABLE a(i integer);
 COPY 1 RECORDS INTO a FROM STDIN USING DELIMITERS ',','\\n','"';
@@ -12,7 +30,7 @@ 42
 COMMIT;
 """
 
-doesnotwork = """-- some comment
+create2 = """-- some comment
 START TRANSACTION;
 CREATE TABLE b(i integer);
 COPY 1 RECORDS INTO b FROM STDIN USING DELIMITERS ',','\\n','"';
@@ -24,19 +42,21 @@ 84
 COMMIT;
 """
 
-cn = mapi.Connection()
+cn = connect()
+query(cn, create1);
+cn.close()
 
-open(cn)
-print(cn.cmd('s'+works+';\n'))
-cn.disconnect()
+cn = connect()
+query(cn, create2);
+cn.close()
 
-open(cn)
-print(cn.cmd('s'+doesnotwork+';\n'))
-cn.disconnect()
 
-# this should show a-e, but only shows a
-open(cn)
-print(cn.cmd('sSELECT name FROM tables WHERE system=0;\n'))
-print(cn.cmd('sDROP TABLE a;\n'))
+# this should show a-c, but only shows a
+cn = connect()
+#print()
 
-cn.disconnect()
+for x in query(cn, 'SELECT name FROM tables WHERE system=0', True):
+print x[0]
+query(cn, 'DROP TABLE a')
+
+cn.close()
diff --git a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out 
b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out
--- a/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out
+++ b/sql/test/BugTracker-2014/Tests/copy-into.Bug-3481.stable.out
@@ -17,6 +17,8 @@ stdout of test 'copy-into.Bug-3481` in d
 # Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-26163/.s.monetdb.33748
 # MonetDB/SQL module loaded
 
+
+
 Ready.
 # SQL catalog created, loading sql scripts once
 # loading sql script: 09_like.sql
@@ -44,35 +46,15 @@ Ready.
 # loading sql script: 80_udf.sql
 # loading sql script: 99_system.sql
 
-# 15:05:54 >  
-# 15:05:54 >  "/usr/bin/python2" "copy-into.Bug-3481.SQL.py" 
"copy-into.Bug-3481"
-# 15:05:54 >  
+# 14:08:38 >  
+# 14:08:38 >  "/usr/local/opt/python/bin/python2.7" 
"copy-into.Bug-3481.SQL.py" "copy-into.Bug-3481"
+# 14:08:38 >  
 
-&4 f
-&3
-&2 1 -1
-&4 t
+a
+b
+c
 
-&4 f
-&3
-&2 1 -1
-&3
-&2 2 -1
-&4 t
+# 14:08:39 >  
+# 14:08:39 >  "Done."
+# 14:08:39 >  
 
-&1 0 3 1 3
-% .tables # table_name
-% name # name
-% varchar # type
-% 1 # length
-[ "a"  ]
-[ "b"  ]
-[ "c"  ]
-
-&3
-
-
-# 15:05:54 >  
-# 15:05:54 >  "Done."
-# 15:05:54 >  
-
diff --git a/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out 
b/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
--- a/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
+++ b/sql/test/BugTracker-2015/Tests/string_split.Bug-3564.stable.out
@@ -61,25 +61,25 @@ Ready.
 # 15:17:35 >  
 
 #select split_part('joeuser@mydatabase','@',1) AS "joeuser";
-% .L1 # table_name
+% .L2 # table_name
 % joeuser # name
 % char # type
 % 18 # length
 [ "joeuser"]
 #select split_part('joeuser@mydatabase','@',2) AS "mydatabase";
-% .L1 # table_name
+% .L2 # table_name
 % mydatabase # name
 % char # type
 % 18 # length
 [ "mydatabase" ]
 #select

MonetDB: queryid2 - merge with default. PAAAANOS :)

2017-02-16 Thread Hannes Muehleisen

Changeset: d9d7507d0a5f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d9d7507d0a5f
Added Files:
sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.sql
sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.stable.err
sql/test/BugTracker-2017/Tests/nested_with.Bug-6187.stable.out
sql/test/BugTracker-2017/Tests/prepare.Bug-6133.sql
sql/test/BugTracker-2017/Tests/prepare.Bug-6133.stable.err
sql/test/BugTracker-2017/Tests/prepare.Bug-6133.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith01.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith02.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith03.stable.err
sql/test/BugTracker-2017/Tests/sqlsmith04.stable.err
Modified Files:
gdk/gdk.h
gdk/gdk_batop.c
gdk/gdk_join.c
gdk/gdk_orderidx.c
gdk/gdk_private.h
gdk/gdk_search.c
gdk/gdk_select.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_runtime.c
monetdb5/modules/mal/orderidx.c
sql/server/sql_parser.y
sql/storage/bat/bat_storage.c
sql/test/BugTracker-2016/Tests/All
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/sqlsmith01.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith04.stable.out
sql/test/emptydb/updatetests
sql/test/lateral/Tests/lateral.sql
Branch: queryid2
Log Message:

merge with default. PNOS :)


diffs (truncated from 2109 to 300 lines):

diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -587,8 +587,6 @@ typedef size_t BUN;
 #else
 #define BUN_NONE ((BUN) LLONG_MAX)
 #endif
-#define BUN_MSK (~BUN_NONE)
-#define BUN_UNMSK BUN_NONE
 #define BUN_MAX (BUN_NONE - 1) /* maximum allowed size of a BAT */
 
 #define BUN2 2
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -1729,6 +1729,8 @@ BATcount_no_nil(BAT *b)
return cnt;
 }
 
+/* create a new, dense candidate list with values from `first' up to,
+ * but not including, `last' */
 static BAT *
 newdensecand(oid first, oid last)
 {
@@ -1738,7 +1740,7 @@ newdensecand(oid first, oid last)
return NULL;
if (last < first)
first = last = 0; /* empty range */
-   BATsetcount(bn, last - first + 1);
+   BATsetcount(bn, last - first);
BATtseqbase(bn, first);
return bn;
 }
@@ -1790,19 +1792,19 @@ BATmergecand(BAT *a, BAT *b)
if (af <= bf && bf <= al + 1) {
/* partial overlap starting with a, or b is
 * smack bang after a */
-   return newdensecand(af, al < bl ? bl : al);
+   return newdensecand(af, al < bl ? bl + 1 : al + 1);
}
if (bf <= af && af <= bl + 1) {
/* partial overlap starting with b, or a is
 * smack bang after b */
-   return newdensecand(bf, al < bl ? bl : al);
+   return newdensecand(bf, al < bl ? bl + 1 : al + 1);
}
}
if (ad && af <= bf && al >= bl) {
-   return newdensecand(af, al);
+   return newdensecand(af, al + 1);
}
if (bd && bf <= af && bl >= al) {
-   return newdensecand(bf, bl);
+   return newdensecand(bf, bl + 1);
}
 
bn = COLnew(0, TYPE_oid, BATcount(a) + BATcount(b), TRANSIENT);
@@ -1912,7 +1914,7 @@ BATintersectcand(BAT *a, BAT *b)
 
if ((af + BATcount(a) - 1 == al) && (bf + BATcount(b) - 1 == bl)) {
/* both lists are VOID */
-   return newdensecand(MAX(af, bf), MIN(al, bl));
+   return newdensecand(MAX(af, bf), MIN(al, bl) + 1);
}
 
bn = COLnew(0, TYPE_oid, MIN(BATcount(a), BATcount(b)), TRANSIENT);
diff --git a/gdk/gdk_join.c b/gdk/gdk_join.c
--- a/gdk/gdk_join.c
+++ b/gdk/gdk_join.c
@@ -182,251 +182,6 @@ joininitresults(BAT **r1p, BAT **r2p, BU
 s##vals + ((x) * s##width))
 #define FVALUE(s, x)   (s##vals + ((x) * s##width))
 
-#define BINSEARCHFUNC(TYPE)\
-static inline BUN  \
-binsearch_##TYPE(const oid *rcand, oid offset, const TYPE *rvals,  \
-BUN lo, BUN hi, TYPE v, int ordering, int last)\
-{  \
-   BUN mid;\
-   TYPE x; \
-   \
-   assert(ordering == 1 || ordering == -1);\
-   assert(lo <= hi);   \
-   \
-

MonetDB: queryid2 - approved test output

2017-02-14 Thread Hannes Muehleisen

Changeset: 72179ed3f8bd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=72179ed3f8bd
Modified Files:

sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out

sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out
sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
sql/test/Tests/identifiers.stable.out
sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py
sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out
sql/test/VOC/Tests/median.Bug-3096.stable.out
sql/test/pg_regress/Tests/date.stable.out
sql/test/pg_regress/Tests/strings.stable.out
sql/test/pg_regress/Tests/strings_concat.stable.out
Branch: queryid2
Log Message:

approved test output


diffs (truncated from 354 to 300 lines):

diff --git 
a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out
 
b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out
--- 
a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out
+++ 
b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out
@@ -36,7 +36,7 @@ Ready.
 % .L12 # table_name
 % suma # name
 % bigint # type
-% 20 # length
+% 1 # length
 [ NULL ]
 #SELECT SUM(a) AS suma
 #FROM ( 
diff --git 
a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
 
b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
--- 
a/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
+++ 
b/sql/test/BugTracker-2009/Tests/TribooleanFailureWithSubquery.SF-2679434.stable.out.int128
@@ -36,7 +36,7 @@ Ready.
 % .L12 # table_name
 % suma # name
 % hugeint # type
-% 40 # length
+% 1 # length
 [ NULL ]
 #SELECT SUM(a) AS suma
 #FROM ( 
diff --git 
a/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out 
b/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out
--- a/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out
+++ b/sql/test/BugTracker-2009/Tests/set_and_arg_null.SF-2812729.stable.out
@@ -34,7 +34,7 @@ Ready.
 % .L2 # table_name
 % L2 # name
 % int # type
-% 11 # length
+% 1 # length
 [ NULL ]
 #create function call_function( aa int ) RETURNS int
 #begin
@@ -46,7 +46,7 @@ Ready.
 % .L2 # table_name
 % L2 # name
 % int # type
-% 11 # length
+% 1 # length
 [ NULL ]
 
 # 22:37:37 >  
diff --git a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out 
b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
--- a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
+++ b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
@@ -37,7 +37,7 @@ Ready.
 [ "€", 1   ]
 #select 'Liever €uro' as "Liever euro";
 % .L2 # table_name
-% Liever euro # name
+% "Liever euro" # name
 % char # type
 % 11 # length
 [ "Liever €uro"]
diff --git a/sql/test/Tests/identifiers.stable.out 
b/sql/test/Tests/identifiers.stable.out
--- a/sql/test/Tests/identifiers.stable.out
+++ b/sql/test/Tests/identifiers.stable.out
@@ -58,13 +58,13 @@ Ready.
 #CREATE TABLE "B\"la\"" (id int); -- should fail
 #SELECT 1 AS "B\"la\""; -- should fail
 % .L2 # table_name
-% B\"la\" # name
+% "B\\\"la\\\"" # name
 % tinyint # type
 % 1 # length
 [ 1]
 #SELECT 1 AS "\"Bla\""; -- should fail
 % .L2 # table_name
-% \"Bla\" # name
+% "\\\"Bla\\\"" # name
 % tinyint # type
 % 1 # length
 [ 1]
diff --git a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py 
b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py
--- a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py
+++ b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.SQL.py
@@ -19,7 +19,7 @@ def query(conn, sql):
 try:
 cur.execute(sql)
 except pymonetdb.OperationalError, e:
-print e
+print "!", e
 return
 r = cur.fetchall()
 cur.close()
@@ -30,9 +30,10 @@ def run(conn, sql):
 try:
 r = conn.execute(sql)
 except pymonetdb.OperationalError, e:
-print e
+print "!", e
 return
-print(r)
+print '# OK'
+
 
 c1 = connect('monetdb', 'monetdb')
 # Create a new schema with a new table.
diff --git a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out 
b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out
--- a/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out
+++ b/sql/test/Users/Tests/grantAndRevokeUserLogedIN.Bug-3476.stable.out
@@ -59,57 +59,43 @@ Ready.
 # 10:44:34 >  
 
 CREATE SCHEMA new_schema_as_well
-&3
-
+# OK
 SET SCHEMA new_schema_as_well
-&4 t
-
+# OK
 CREATE TABLE test (x int, y int)
-&3
-
+# OK
 INSERT INTO test VALUES (-1, -1)
-&2 1 -1
-
+# OK
 CREATE USER

MonetDB: queryid2 - merge with default

2017-02-14 Thread Hannes Muehleisen

Changeset: 3de277fe54c3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=3Dchangeset;node=3D3de277fe54c3
Removed Files:

sql/test/BugTracker-2010/Tests/group-by_ordered_column.Bug-2564.stable.out=
.32bit
Modified Files:
monetdb5/mal/mal_function.c
monetdb5/mal/mal_profiler.c
monetdb5/tools/Tests/mserver5--help.bat
monetdb5/tools/Tests/mserver5--help.stable.err.Windows
sql/backends/monet5/UDF/pyapi/convert_loops.h

sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Darwin

sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err.Windows
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/sqlsmith02.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith03.stable.out
Branch: queryid2
Log Message:

merge with default


diffs (truncated from 414 to 300 lines):

diff --git a/monetdb5/mal/mal_function.c b/monetdb5/mal/mal_function.c
--- a/monetdb5/mal/mal_function.c
+++ b/monetdb5/mal/mal_function.c
@@ -924,7 +924,7 @@ showFlowDetails(MalBlkPtr mb, MalStkPtr=20
 {
(void) mb; /* fool the compiler */
(void) stk; /* fool the compiler */
-   mnstr_printf(f, "n%d [fontsize=3D8, shape=3Dbox, label=3D\"%s\"]\n", 
pc, =
getFunctionId(p));
+   mnstr_printf(f, "n%d [fontsize=3D8, shape=3Dbox, label=3D\"%s\"]\n", 
pc, =
getFunctionId(p) ? getFunctionId(p) : "");
 }
=20
 /* Produce a file with the flow graph in dot format.
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -125,7 +125,7 @@ renderProfilerEvent(MalBlkPtr mb, MalStk
if( usrname)
logadd("\"user\":\"%s\",%s",usrname, prettify);
logadd("\"clk\":"LLFMT",%s",usec,prettify);
-   logadd("\"ctime\":"LLFMT".%06ld\",%s", clock / 100, (long) (clock % 
1=
00), prettify);
+   logadd("\"ctime\":"LLFMT".%06ld,%s", clock / 100, (long) (clock % 
100=
), prettify);
logadd("\"thread\":%d,%s", THRgettid(),prettify);
=20
logadd("\"function\":\"%s.%s\",%s", getModuleId(getInstrPtr(mb, 0)), 
getF=
unctionId(getInstrPtr(mb, 0)), prettify);
@@ -400,7 +400,7 @@ profilerHeartbeatEvent(char *alter)
lognew();
logadd("{%s",prettify); // fill in later with the event counter
logadd("\"user\":\"heartbeat\",%s", prettify);
-   logadd("\"ctime\":"LLFMT".%06ld\",%s", clock / 100, (long) (clock % 
1=
00), prettify);
+   logadd("\"ctime\":"LLFMT".%06ld,%s", clock / 100, (long) (clock % 
100=
), prettify);
logadd("\"rss\":"SZFMT ",%s", MT_getrss()/1024/1024, prettify);
 #ifdef HAVE_SYS_RESOURCE_H
getrusage(RUSAGE_SELF, );
diff --git a/monetdb5/tools/Tests/mserver5--help.bat b/monetdb5/tools/Tests=
/mserver5--help.bat
--- a/monetdb5/tools/Tests/mserver5--help.bat
+++ b/monetdb5/tools/Tests/mserver5--help.bat
@@ -1,4 +1,4 @@
 @echo on
 @prompt # $t $g =20
=20
-   %MTIMEOUT% mserver5 --help
+   %MTIMEOUT% mserver5.exe --help
diff --git a/monetdb5/tools/Tests/mserver5--help.stable.err.Windows b/monet=
db5/tools/Tests/mserver5--help.stable.err.Windows
--- a/monetdb5/tools/Tests/mserver5--help.stable.err.Windows
+++ b/monetdb5/tools/Tests/mserver5--help.stable.err.Windows
@@ -5,11 +5,6 @@ stderr of test 'mserver5--help` in direc
 # 16:36:12 >  Mtimeout -timeout 300 ./mserver5--help mserver5--help=20
 # 16:36:12 > =20
=20
-
-# 16:36:12 > =20
-# 16:36:12 >  Mtimeout -timeout 60 mserver5 --config=3D --help
-# 16:36:12 > =20
-
 Usage: mserver5.exe [options] [scripts]
 --dbpath=3D  Specify database location
 --dbextra=3D Directory for transient BATs
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h b/sql/backends/m=
onet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -24,6 +24,37 @@
=20
 // This #define creates a new BAT with the internal data and mask from a N=
umpy array, without copying the data
 // 'bat' is a BAT* pointer, which will contain the new BAT. TYPE_'mtpe' is=
 the BAT type, and 'batstore' is the heap storage type of the BAT (this sho=
uld be STORE_CMEM or STORE_SHARED)
+#if defined(_MSC_VER) && _MSC_VER <=3D 1600
+#define isnan(x) _isnan(x)
+#endif
+
+#define nancheck_flt(bat)  
\
+   do {\
+   for (iu =3D 0; iu < ret->count; iu++) { \
+   if (isnan(((flt*)data)[index_offset * ret->count + 
iu])) { \
+   ((flt*)data)[index_offset * ret->count + iu] 
=3D flt_nil; \
+   bat->tnil =3D 1;
\
+   }   \
+   }   \
+

MonetDB: queryid2 - merge with default

2017-02-14 Thread Hannes Muehleisen

Changeset: 0210c5d6072b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=3Dchangeset;node=3D0210c5d6072b
Modified Files:
MonetDB.spec
geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
monetdb5/modules/mal/mkey.c
sql/backends/monet5/sql.c

sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.stable=
.err
sql/test/BugTracker-2008/Tests/copy-format.SF-2468513.stable.err
sql/test/BugTracker-2008/Tests/varchar.SF-2471371.stable.err

sql/test/BugTracker-2009/Tests/copy_into_error_off_by_one.SF-2833696.stabl=
e.err

sql/test/BugTracker-2010/Tests/copy-into-too-long-string.Bug-2358.stable.e=
rr
sql/test/BugTracker-2010/Tests/error-truncated.Bug-2615.stable.err

sql/test/BugTracker-2010/Tests/incomplete-utf8-sequence.Bug-2575.stable.err

sql/test/BugTracker-2015/Tests/backslash_predecing_separator.Bug-3775.stab=
le.err
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker/Tests/copy_decimal.SF-1582957.stable.err
sql/test/BugTracker/Tests/copy_overflow_null_bug.SF-1842550.stable.err
sql/test/copy/Tests/columns.stable.err
sql/test/copy/Tests/int_parse.stable.err
sql/test/copy/Tests/load_stdin_incorrect_line_nr.stable.err
sql/test/copy/Tests/null_as_string_errors.stable.err
Branch: queryid2
Log Message:

merge with default


diffs (truncated from 315 to 300 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -201,6 +201,7 @@ package, and most likely also %{name}-SQ
 more client packages.
=20
 %files
+%license COPYING
 %defattr(-,root,root)
 %{_libdir}/libbat.so.*
=20
@@ -242,6 +243,7 @@ This package contains a shared library (
 various other components.
=20
 %files stream
+%license COPYING
 %defattr(-,root,root)
 %{_libdir}/libstream.so.*
=20
@@ -289,6 +291,7 @@ SQL database so that it can be loaded ba
 MonetDB, you will very likely need this package.
=20
 %files client
+%license COPYING
 %defattr(-,root,root)
 %{_bindir}/mclient
 %{_bindir}/msqldump
@@ -375,6 +378,7 @@ odbcinst -u -d -n MonetDB
 fi
=20
 %files client-odbc
+%license COPYING
 %defattr(-,root,root)
 %{_libdir}/libMonetODBC.so
 %{_libdir}/libMonetODBCs.so
@@ -853,6 +857,7 @@ MonetDB packages.  You probably don't ne
 developer.  If you do want to test, install %{name}-testing-python.
=20
 %files testing
+%license COPYING
 %defattr(-,root,root)
 %{_bindir}/Mdiff
 %{_bindir}/MkillUsers
diff --git a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err b/=
geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
--- a/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
+++ b/geom/BugTracker/Tests/copy_into_crash.SF-1975402.stable.err
@@ -13,7 +13,7 @@ stderr of test 'copy_into_crash.SF-19754
 MAPI  =3D (monetdb) /var/tmp/mtest-27003/.s.monetdb.30235
 QUERY =3D COPY 1 RECORDS INTO nodes from STDIN USING DELIMITERS ',', '\n';
 45111956, 'POINT(52.0697 4.3723)'
-ERROR =3D !Failed to import table line 1 field p 'geometry(4)' expected in=
 ' 'POINT(52.0697 4.3723)''
+ERROR =3D !Failed to import table 'nodes', line 1 field p 'geometry(4)' ex=
pected in ' 'POINT(52.0697 4.3723)''
=20
 # 13:33:18 > =20
 # 13:33:18 >  Done.
diff --git a/monetdb5/modules/mal/mkey.c b/monetdb5/modules/mal/mkey.c
--- a/monetdb5/modules/mal/mkey.c
+++ b/monetdb5/modules/mal/mkey.c
@@ -50,6 +50,11 @@ MKEYhash(Client cntxt, MalBlkPtr mb, Mal
res=3D getArgReference_lng(stk,p,0);
val=3D getArgReference(stk,p,1);
switch (ATOMstorage(tpe)) {
+   case TYPE_void:
+   case TYPE_bat:
+   case TYPE_ptr:
+   // illegal types, avoid falling into the default case.
+   assert(0);
case TYPE_bte:
*res =3D MKEYHASH_bte(val);
break;
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -2507,7 +2507,7 @@ mvc_import_table_wrap(Client cntxt, MalB
if (fname && s =3D=3D NULL)
throw(IO, "bstreams.create", "Failed to create block stream");
if (b =3D=3D NULL)
-   throw(SQL, "importTable", "Failed to import table %s", 
be->mvc->errstr);
+   throw(SQL, "importTable", "Failed to import table '%s', %s", 
t->base.nam=
e, be->mvc->errstr);
bat2return(stk, pci, b);
GDKfree(b);
return msg;
diff --git a/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-11=
00504.stable.err b/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into=
.SF-1100504.stable.err
--- a/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.s=
table.err
+++ b/sql/test/BugDay_2005-11-09_2.9.3/Tests/hang_on_copy_into.SF-1100504.s=
table.err
@@ -88,9 +88,7 @@ QUERY =3D COPY 1 RECORDS INTO "news" FROM=20
 allemaal nogal lang duurt, maar in principe is het nog
 maar een uurtje of 6 werk om het goed te krijgen. Zoals

MonetDB: queryid2 - merge with default

2017-02-13 Thread Hannes Muehleisen

Changeset: c224e8a2b82a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c224e8a2b82a
Added Files:
monetdb5/extras/rapi/Tests/rapi02.stable.out.int128
sql/backends/monet5/Tests/int_notation_1e5.stable.out.int128
sql/backends/monet5/UDF/pyapi/Tests/pyapi_types_huge.stable.out.int128
sql/jdbc/tests/Tests/Test_Int128.stable.out.int128

sql/test/BugTracker-2015/Tests/hugeint-wrong-value.Bug-3849.stable.out.int128

sql/test/BugTracker-2016/Tests/convert-function-test-hge.Bug-3460.stable.out.int128

sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out.int128
sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.sql
sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.stable.err
sql/test/BugTracker-2017/Tests/crash_on_NULL_ptr.Bug-6130.stable.out
sql/test/BugTracker-2017/Tests/dce_bug-6177.sql
sql/test/BugTracker-2017/Tests/dce_bug-6177.stable.err
sql/test/BugTracker-2017/Tests/dce_bug-6177.stable.out

sql/test/BugTracker-2017/Tests/group-by-on-constant.Bug-6082.stable.out.int128
sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.sql

sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.stable.err

sql/test/BugTracker-2017/Tests/integer_addition_overflow.Bug-6205.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith01.sql
sql/test/BugTracker-2017/Tests/sqlsmith01.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith02.sql
sql/test/BugTracker-2017/Tests/sqlsmith02.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith03.sql
sql/test/BugTracker-2017/Tests/sqlsmith03.stable.out
sql/test/BugTracker-2017/Tests/sqlsmith04.sql
sql/test/BugTracker-2017/Tests/sqlsmith04.stable.out
sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.sql
sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.stable.err
sql/test/BugTracker-2017/Tests/str2decimal.Bug-6206.stable.out
sql/test/emptydb/Tests/check.SQL.py
sql/test/mapi/Tests/mal_int128.stable.out.int128
sql/test/mapi/Tests/perl_dec38.stable.out.int128
sql/test/mapi/Tests/perl_int128.stable.out.int128
sql/test/mapi/Tests/php_dec38.stable.out.int128
sql/test/mapi/Tests/php_int128.stable.out.int128
sql/test/mapi/Tests/python2_dec38.stable.out.int128
sql/test/mapi/Tests/python2_int128.stable.out.int128
sql/test/mapi/Tests/python3_dec38.stable.out.int128
sql/test/mapi/Tests/python3_int128.stable.out.int128
sql/test/mapi/Tests/sql_dec38.stable.out.int128
sql/test/mapi/Tests/sql_int128.stable.out.int128
sql/test/mergetables/Tests/addtable.sql
sql/test/mergetables/Tests/addtable.stable.err
sql/test/mergetables/Tests/addtable.stable.out
sql/test/pg_regress/Tests/numeric.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/dump.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/package.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/unpackage.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/dump.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/unpackage.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb/Tests/testdb-package-hge.stable.out.int128
Removed Files:
clients/Tests/SQL-dump.SQL.py
clients/Tests/SQL-dump.stable.err
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
gdk/bat.feps
gdk/bat.png
gdk/bat1.feps
gdk/bat1.png
gdk/bat2.feps
gdk/bat2.png
monetdb5/extras/rapi/Tests/rapi02.stable.out
sql/backends/monet5/Tests/int_notation_1e5.stable.out
sql/backends/monet5/UDF/pyapi/Tests/pyapi_types_huge.stable.out
sql/jdbc/tests/Tests/Test_Int128.stable.out
sql/test/BugTracker-2014/Tests/nil_2dec_lng.Bug-3592.stable.out.single
sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.err.single
sql/test/BugTracker-2014/Tests/querylog.Bug-3607.stable.out.single
sql/test/BugTracker-2015/Tests/hugeint-wrong-value.Bug-3849.stable.out
sql/test/BugTracker-2015/Tests/useless_casts.Bug-3756.stable.out.single

sql/test/BugTracker-2016/Tests/convert-function-test-hge.Bug-3460.stable.out

sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit
sql/test/emptydb/Tests/check.SQL.py.src
sql/test/mapi/Tests/mal_int128.stable.out
sql/test/mapi/Tests/perl_dec38.stable.out
sql/test/mapi/Tests/perl_int128.stable.out
sql/test/mapi/Tests/php_dec38.stable.out

MonetDB: queryid2 - approve test output

2017-02-07 Thread Hannes Muehleisen

Changeset: 6939acb0621d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6939acb0621d
Modified Files:

sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out
sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out
Branch: queryid2
Log Message:

approve test output


diffs (70 lines):

diff --git 
a/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out 
b/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out
--- a/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out
+++ b/sql/test/BugDay_2005-12-19_2.9.3/Tests/NULL_in_CASE.SF-1211335.stable.out
@@ -25,40 +25,40 @@ Ready.
 # 11:50:59 >  
 
 #select CASE WHEN 1=1 THEN 1 ELSE NULL END;
-% .L1 # table_name
-% L1 # name
+% .L2 # table_name
+% L2 # name
 % tinyint # type
 % 1 # length
 [ 1]
 #select CASE WHEN 1=0 THEN 1 ELSE NULL END;
-% .L1 # table_name
-% L1 # name
+% .L2 # table_name
+% L2 # name
 % tinyint # type
-% 4 # length
+% 1 # length
 [ NULL ]
 #select CASE WHEN 2=2 THEN 2 else cast(null as char) end;
-% .L2 # table_name
-% L2 # name
+% .L3 # table_name
+% L3 # name
 % char # type
 % 1 # length
 [ "2"  ]
 #select CASE WHEN 2=0 THEN 2 else cast(null as char) end;
-% .L2 # table_name
-% L2 # name
+% .L3 # table_name
+% L3 # name
 % char # type
 % 1 # length
 [ NULL ]
 #select CASE WHEN 3=3 THEN 3 else cast(null as int) end;
-% .L2 # table_name
-% L2 # name
+% .L3 # table_name
+% L3 # name
 % int # type
 % 1 # length
 [ 3]
 #select CASE WHEN 3=0 THEN 3 else cast(null as int) end;
-% .L2 # table_name
-% L2 # name
+% .L3 # table_name
+% L3 # name
 % int # type
-% 11 # length
+% 1 # length
 [ NULL ]
 
 # 11:35:46 >  
diff --git a/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out 
b/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out
--- a/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out
+++ b/sql/test/BugTracker-2014/Tests/ifthenelse.Bug-3629.stable.out
@@ -56,7 +56,7 @@ Ready.
 % .L2 # table_name
 % L2 # name
 % int # type
-% 11 # length
+% 1 # length
 [ NULL ]
 
 # 18:33:32 >  
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: queryid2 - queryid re-implementation

2017-02-07 Thread Hannes Muehleisen

Changeset: 6774196d9fba for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6774196d9fba
Modified Files:
monetdb5/mal/mal_runtime.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_execute.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
sql/include/sql_catalog.h
sql/storage/bat/res_table.c
sql/storage/sql_storage.h
Branch: queryid2
Log Message:

queryid re-implementation


diffs (287 lines):

diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c
--- a/monetdb5/mal/mal_runtime.c
+++ b/monetdb5/mal/mal_runtime.c
@@ -82,6 +82,7 @@ runtimeProfileInit(Client cntxt, MalBlkP
// add new invokation
QRYqueue[i].mb = mb;
QRYqueue[i].tag = qtag++;
+   mb->tag = QRYqueue[i].tag;
QRYqueue[i].stk = stk;  // for status pause 
'p'/running '0'/ quiting 'q'
QRYqueue[i].start = (lng)time(0);
QRYqueue[i].runtime = mb->runtime;  // the estimated execution time
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -1705,7 +1705,7 @@ mvc_result_set_wrap( Client cntxt, MalBl
b = BATdescriptor(bid);
if ( b == NULL)
throw(MAL,"sql.resultset","Failed to access order column");
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, b);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
5), 1, b);
if (res < 0)
msg = createException(SQL, "sql.resultSet", "failed");
BBPunfix(b->batCacheid);
@@ -1796,7 +1796,7 @@ mvc_export_table_wrap( Client cntxt, Mal
order = BATdescriptor(bid);
if ( order == NULL)
throw(MAL,"sql.resultset","Failed to access order column");
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, 
order);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
11), 1, order);
t = m->results;
if (res < 0){
msg = createException(SQL, "sql.resultSet", "failed");
@@ -1895,7 +1895,7 @@ mvc_row_result_wrap( Client cntxt, MalBl
return msg;
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 5), 1, 
NULL);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
5), 1, NULL);
 
tbl = BATdescriptor(tblId);
atr = BATdescriptor(atrId);
@@ -1970,7 +1970,7 @@ mvc_export_row_wrap( Client cntxt, MalBl
return msg;
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
-   res = *res_id = mvc_result_table(m, pci->argc - (pci->retc + 11), 1, 
NULL);
+   res = *res_id = mvc_result_table(m, mb->tag, pci->argc - (pci->retc + 
11), 1, NULL);
 
t = m->results;
if (res < 0){
@@ -2071,7 +2071,7 @@ mvc_table_result_wrap(Client cntxt, MalB
if ((order = BATdescriptor(*order_bid)) == NULL) {
throw(SQL, "sql.resultSet", "Cannot access descriptor");
}
-   *res_id = mvc_result_table(m, *nr_cols, *qtype, order);
+   *res_id = mvc_result_table(m, mb->tag, *nr_cols, *qtype, order);
if (*res_id < 0)
res = createException(SQL, "sql.resultSet", "failed");
BBPunfix(order->batCacheid);
@@ -2203,7 +2203,7 @@ mvc_affected_rows_wrap(Client cntxt, Mal
assert(mtype == TYPE_lng);
nr = *getArgReference_lng(stk, pci, 2);
b = cntxt->sqlcontext;
-   error = mvc_export_affrows(b, b->out, nr, "");
+   error = mvc_export_affrows(b, b->out, nr, "", mb->tag);
if (error)
throw(SQL, "sql.affectedRows", "failed");
return MAL_SUCCEED;
@@ -2301,12 +2301,11 @@ mvc_scalar_value_wrap(Client cntxt, MalB
str *type = getArgReference_str(stk, pci, 3);
int *digits = getArgReference_int(stk, pci, 4);
int *scale = getArgReference_int(stk, pci, 5);
-   int *eclass = getArgReference_int(stk, pci, 6);
ptr p = getArgReference(stk, pci, 7);
int mtype = getArgType(mb, pci, 7);
str msg;
backend *b = NULL;
-
+   int res_id;
(void) mb;  /* NOT USED */
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
@@ -2315,13 +2314,14 @@ mvc_scalar_value_wrap(Client cntxt, MalB
p = *(ptr *) p;
 
// scalar values are single-column result sets
-   mvc_result_table(b->mvc, 1, 1, NULL);
+   res_id = mvc_result_table(b->mvc, mb->tag, 1, 1, NULL);
mvc_result_value(b->mvc, *tn, *cn, *type, *digits, *scale, p, mtype);
if (b->output_format == OFMT_NONE) {
return MAL_SUCCEED;
}
-   if (b->out == NULL || mvc_export_value(b, b->out, 1, *tn, *cn, *type, 
*digits, *scale, *eclass, p, mtype, "",

MonetDB: queryid - closing branch dude

2017-02-07 Thread Hannes Muehleisen

Changeset: 614d7bbd027b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=614d7bbd027b
Branch: queryid
Log Message:

closing branch dude

___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: queryid - merge with default

2017-02-01 Thread Hannes Muehleisen

Changeset: 312f53b33312 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=312f53b33312
Added Files:
README.rst
clients/Tests/exports.stable.err-noapprove
clients/Tests/malcheck.stable.err-noapprove
clients/Tests/malcheck.stable.out-noapprove
debian/monetdb-python2.install
monetdb5/modules/kernel/Tests/math.malC
monetdb5/modules/kernel/Tests/math.stable.err
monetdb5/modules/kernel/Tests/math.stable.out
monetdb5/modules/mal/oltp.c
monetdb5/modules/mal/oltp.h
monetdb5/modules/mal/oltp.mal
monetdb5/optimizer/opt_oltp.c
monetdb5/optimizer/opt_oltp.h
monetdb5/tools/Tests/mserver5--help.stable.err.Windows
sql/backends/monet5/Tests/createorreplace.sql
sql/backends/monet5/Tests/createorreplace.stable.err
sql/backends/monet5/Tests/createorreplace.stable.out
sql/backends/monet5/Tests/pyapi33.sql
sql/backends/monet5/Tests/pyapi33.stable.err
sql/backends/monet5/Tests/pyapi33.stable.out
sql/backends/monet5/sql_cast_impl_int.h
sql/backends/monet5/sql_cat.c
sql/backends/monet5/sql_cat.h
sql/backends/monet5/sql_transaction.c
sql/backends/monet5/sql_transaction.h
sql/backends/monet5/sql_transaction.mal
sql/backends/monet5/sqlcatalog.mal
sql/test/BugTracker-2015/Tests/long_insert.Bug-3845.timeout
sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.sql
sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.err
sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.out

sql/test/BugTracker-2016/Tests/inner_join_cross_join.Bug-6140.stable.out.int128
sql/test/BugTracker-2017/Tests/All
sql/test/BugTracker-2017/Tests/all_dates.csv.gz
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.sql

sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.err

sql/test/BugTracker-2017/Tests/cast-with-many-decimals.Bug-6195.stable.out
sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.sql
sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.stable.err
sql/test/BugTracker-2017/Tests/empty-interval.Bug-6184.stable.out
sql/test/BugTracker-2017/Tests/heapextend.Bug-6134.sql.in
sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.sql
sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.stable.err
sql/test/BugTracker-2017/Tests/round-or-truncate.Bug-6193.stable.out
sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.sql
sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.stable.err
sql/test/BugTracker-2017/Tests/semijoinunion.Bug-6150.stable.out
sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.sql
sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.stable.err
sql/test/BugTracker-2017/Tests/shutdown.Bug-6182.stable.out
sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.sql
sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.err
sql/test/BugTracker-2017/Tests/too-large-decimals.Bug-6192.stable.out
sql/test/Tests/bincopycollist-prepare.py
sql/test/Tests/ifexists.sql
sql/test/Tests/ifexists.stable.err
sql/test/Tests/ifexists.stable.out
sql/test/Tests/subqueries.sql
sql/test/Tests/subqueries.stable.err
sql/test/Tests/subqueries.stable.out
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out-noapprove
sql/test/emptydb-upgrade-chain-hge/Tests/dump.stable.err-noapprove
sql/test/emptydb-upgrade-chain-hge/Tests/dump.stable.out-noapprove
sql/test/emptydb-upgrade-chain/Tests/check.stable.out-noapprove
sql/test/emptydb-upgrade-chain/Tests/dump.stable.err-noapprove
sql/test/emptydb-upgrade-chain/Tests/dump.stable.out-noapprove
sql/test/emptydb-upgrade-hge/Tests/check.stable.out-noapprove
sql/test/emptydb-upgrade-hge/Tests/dump.stable.err-noapprove
sql/test/emptydb-upgrade-hge/Tests/dump.stable.out-noapprove
sql/test/emptydb-upgrade/Tests/check.stable.out-noapprove
sql/test/emptydb-upgrade/Tests/dump.stable.err-noapprove
sql/test/emptydb-upgrade/Tests/dump.stable.out-noapprove
sql/test/emptydb/Tests/dump.stable.err-noapprove
sql/test/emptydb/Tests/dump.stable.out-noapprove
sql/test/malloc_fail/Tests/mallocs.SQL.py
sql/test/sys-schema/Tests/All
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.sql
sql/test/sys-schema/Tests/check_AlternateKey_uniqueness.stable.err

MonetDB: Dec2016 - new files, Bug 6134

2017-01-10 Thread Hannes Muehleisen

Changeset: f0eaa26022fe for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f0eaa26022fe
Added Files:
sql/test/BugTracker-2017/Tests/all_dates.csv.gz
sql/test/BugTracker-2017/Tests/heapextend.Bug-6134.sql.in
Branch: Dec2016
Log Message:

new files, Bug 6134


diffs (truncated from 1704 to 300 lines):

diff --git a/sql/test/BugTracker-2017/Tests/all_dates.csv.gz 
b/sql/test/BugTracker-2017/Tests/all_dates.csv.gz
new file mode 100644
index 
e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5b660b69ded583f9ba5235a6e32aa016a31820e7
GIT binary patch
literal 92846
zc%1EBeNau$TFi-oly1ZS~Rk+BYbqBL@Mt`u9TUY8cUBGN5Y>F{-P5yOyZ
z#yWG=6q0SHYi$+SrBn14fnpI52(ed_%+@MH{XhvMNT!0Yd_zC_|Iy!VT_oM`@#1;!l)d@((Is17{(8foPdolm
zyJ*ir3djX{55~cPHBD5mkoaU@T-f*J{bGj{ek0aW!b8_+SkwY{NXkKEvikX
zM6v~Ew?UuJ%=i1~rz>-0BjTguYnO)u3*H<|w9wSZ-+WYgBpBZU8+PZu-Scp6;e
zNI-E~UhfgszQjpuzGVKG6!@{z^OhdHnY6bkd}mmqAe<*X)$>(;#CquhQ*lqBmjh`j
z6P+>gfJOH2_Oi+a5|wB2{}XEqPV-@20n#{c}`tg0Pl^lUtcT77P5?+eeRW7>MXd
zlZsHkyBYT>rMvNy(yyYyf37OQ^oQnvP*ss7CoiVV`?;(%hWrgU?xe<*Q{p9K&
zfuF0#K`qL$BpuD_(=o;h`L|M7MlSi;{(y(r}Uk=QIW9Fi(8nJbX+rWM^F+
zRcVZaSnKlg;{eb$3%mg>wJj>k7OmKIa`k~rU*#_@k$%=wX<3|oNtn0dP3BrR0&88;
z_DQ;ZUS6=+Y978hZoToQB#8B{E5C)b-iHB|HD9d)P`<4{fiV-pRwkTj_VU2^+^
zE2857iD-n8h>Uns5V}NkrRAA%Bx1NQd_A&6&?=(Yp)-TE?fw^n(oUUrD6_L
zot77ZA-pedH*`1AD)eTQg=Bp#DYry;Mfd*Lkc7V@t<^5QdFw~;;*Y^-8Cj$%4Ov?x
z?Umgq1}-qA3+CcimFE|2Q?bNd>P_4qN{uy*ph~Ma0ig;D)c<>5x;??T%|
zLn)FiIT^uX32TRv3acN4=n9C9fDw#Vc@z2lv4`L96W!iyuoj-GIJ{422`U5IW_8IHTt~RrqF~Kp0`CXMT$)Bl0)1?QP{@i
zNc4{W$3R4oStca(BE51tJE-LTk~08&}{txac_|RlgbjwR)0yX
z_scw;CIyMc<26odTTt?LK>E9WgwYz2<@p$AbG4>17sm!Ss{N}md;@->P!22-!Y|k#
z+j_ib1*!--&^;Kb0;R=!O-{ZE4qE)2;$NW#tX~5r~^;5H7-SC~0FaEVxWNPe;
zX`|*lGbt))qW>f@MbfrHl?rj0{m0*nL(w?>Nww-@Hv#FdUkT%E7l!jNPaF|$
zu8LG;V9;RsjV>%2yrSHKEP@Vni{b^KRCpKNBf3L`ou+wmUph|dCs*Y}#30kJ
z14l~)L;Gp(sl-R#y)y*$uXUg;{e}xc`8EW#^I$1W^Y(x=57vMB#Tz*Olg>o`)f)
z{!^_cAfHlb1_iT7E(kSayr=Faszf(BRe0_AK7Wt2<`!6AWW$a2pF%kQX`1>^
z)PN+G8h_`L*5)st)AB$>(^@WII+FsXSM=AC)u1m@-xvz9*6)wM=i_+kb-~aey2I(xLUbI6KV8H=tP~4y1N56>cHH~0o9V#hWDVViWLI^UrKKoQ2|=x+^}R(@5Is)qb0Mf{9fF!L(>Q7A=vBcGpZ4>vas?)A8!^p
z6dSs#98PBRRFULVec?4VPG@mj|8XS;Ps!lFdA=wKy)>^Ye*Hu$B4
z*D!2w@3yYBI5s#XP1=qj6}*ZjEUDNNZwf?}3Wu5)J>CNbX1z<3pCYMyfZ!X0lDgTM
zMt%Q*4G=Uq`1hY+B;*=|wkFN4!sy)MJ^VoIUu8dq}gc~3ROW#+}#APMtG#P+mJ
zpe;@Q0)kwuQ~S#?_*5QDGpk8MBR1L}MTjnSy-2mKg08XTA)MixN_y7}b|h@LAg
zx*Ld}$uvSbPG%W~Ce~FWHjQ%7viVwsd-X_dk7~)gKt*kQvdkHr{N#-8@{!CGcmK_e85w?Ns5VrDddFOx{Kd#@V}YDf9_V=V=g-A=-r6^)vD|o>6ada(AG%w_@dLyphzz%u>K5X)>`t=faU%(!7*i(RP5
zixyLtm`iQ#Kj5X#*m30@r0jk4lewOi*M)u$Hoq(TELW8SS#=l9TbMsdu2IE!QS7gm
zJ&-oORg-+CXR#ahvuR=DJ(jGl)E?Un>@K;(%kGLc+z+VCl4fAwC9@(L3oqy2GIgTj
z<$gelwjY$Cs%!|^+nbZnT+d!+v8G
zeQ15%d<$DKoiTdL?tGlyQa1RDH|kIYN*t@zo9=6NS)?dMX7t%mifl*|9b`+D
zB*FHg7h<;Pi|W%_|0dNEnTOH+Mjvi`JX8HP>jUr_zk<2Oj`9YKqtVngrU91-BfJ5{
zTK`rIkvLar?7$HT^JE-V^0oe@$jc0E6IpmvRejo2(WHB`4-*fFg6MkA+RrF!
zKjEE2DpF!2jv_s0%2n)YT8*Pf+xJ;^(n6I13KgeFr?n!T@XjZ#BF#BW*olHBA1P8t
zDr)zFTbV#4;N8bY0>%XR+*^VyqzD52j|Wm!6;Ylr|L?^~^>N;>VqJZB3F6lN
zOfNDcx$7JtwKoPK4$_z9?3?~|F|2`Sa%}@z)VV)H_Xr$
zsZUp)pMxraj%DsS(eo|xGQ$ey>U;GgqnU>Z_MQJ{m$yQWFsBFfF?gg9+JaW1y#<%|
zW!aD~Vw%LAce7ak@3eznS{
zdXzlU7&2kb;VF4dPGf%%H1@OWIAiZyW1rehS`Ceg3li2+_vCq-4(3qZS+8t7i+(cw
zb)-(LtJvuF<|=dxiOv)QO%_2odfozlEgNMk2n*+5{fxQW%%w=4ut_ia?yo
zy1E-7Sa|6y?TcYlzY}#AFe~ToQ`GQF1MiTvm?5TLfUG68`6RWPe~jG
zJUgn2N(Yu|~Sy~(ic+B;43db!BcOk$!UOO>Qu+W|7WZyUW8YdVxgoiYqPA{^
zu5KMUM$=J@e0_-aj>y)elMN2pUu1qwy=B@0VP>)u{*GjO9|l~h`wHCRF_>%2IMD-M
zKg7|5X!~VYG40|A(l^1SRNS8+Cy-B`V7&*YKf%t9LS!P;p9>8qI^9|3Km>FUcV{JZr7~@w%GL}
z#C4=+CG5aBnzZDu8l0H=)huZ*hJV(PCc3#&7@G#rM+Y2+@=|$NGHDYNgEv96D=;Kg=(Ci$0^cjtw<-l^GT{mmu{y-
zbM+%is;Zu1gN<6FQgSK#X`8Yyoo}Hul+{48-MmR#MtXZ^x6Nue%W={frAqG;1((6{
zcDSmNPqU;xjzHx_i>*Y#fRC#2X)UV#5!9laI4$bq!l$7@

MonetDB: Dec2016 - Added test for Bug 6134

2017-01-10 Thread Hannes Muehleisen

Changeset: d21bef4f8fdd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d21bef4f8fdd
Modified Files:
sql/test/BugTracker-2017/Tests/All
Branch: Dec2016
Log Message:

Added test for Bug 6134


diffs (8 lines):

diff --git a/sql/test/BugTracker-2017/Tests/All 
b/sql/test/BugTracker-2017/Tests/All
--- a/sql/test/BugTracker-2017/Tests/All
+++ b/sql/test/BugTracker-2017/Tests/All
@@ -1,3 +1,4 @@
 shutdown.Bug-6182
 avggroupbysq.Bug-6178
 semijoinunion.Bug-6150
+heapextend.Bug-6134
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - merge

2017-01-06 Thread Hannes Muehleisen

Changeset: d314807446e7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d314807446e7
Modified Files:
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_stack.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_reorder.c
sql/backends/monet5/sql_optimizer.c
Branch: Dec2016
Log Message:

merge


diffs (141 lines):

diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c
--- a/monetdb5/mal/mal_builder.c
+++ b/monetdb5/mal/mal_builder.c
@@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val
}
cst.len= (int) strlen(cst.val.sval);
getArg(q,0) = defConstant(mb,TYPE_str,);
+   if (getArg(q,0) < 0) {
+   freeInstruction(q);
+   return NULL;
+   }
clrVarConstant(mb,getArg(q,0));
setVarDisabled(mb,getArg(q,0));
if (mb->errors) {
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -730,7 +730,6 @@ makeVarSpace(MalBlkPtr mb)
if (new == NULL) {
mb->errors++;
showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no 
storage left\n");
-   assert(0);
return -1;
}
memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr));
@@ -1258,6 +1257,9 @@ defConstant(MalBlkPtr mb, int type, ValP
return k;
}
k = newTmpVariable(mb, type);
+   if (k == -1) {
+   return k;
+   }
setVarConstant(mb, k);
setVarFixed(mb, k);
if (type >= 0 && type < GDKatomcnt && ATOMextern(type))
diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c
--- a/monetdb5/mal/mal_interpreter.c
+++ b/monetdb5/mal/mal_interpreter.c
@@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size)
 
assert(size >= mb->vsize);
stk = newGlobalStack(size);
+   if (!stk) {
+   return NULL;
+   }
//memset((char *)stk, 0, stackSize(size)); already set
//stk->stksize = size;
stk->stktop = mb->vtop;
diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c
--- a/monetdb5/mal/mal_stack.c
+++ b/monetdb5/mal/mal_stack.c
@@ -59,8 +59,9 @@ newGlobalStack(int size)
MalStkPtr s;
 
s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk));
-   if (s == NULL)
-   GDKfatal("newGlobalStack:can not obtain memory\n");
+   if (!s) {
+   return NULL;
+   }
s->stksize = size;
return s;
 }
@@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn
return old;
k = ((cnt / STACKINCR) + 1) * STACKINCR;
s = newGlobalStack(k);
+   if (!s) {
+   return NULL;
+   }
memcpy(s, old, stackSize(old->stksize));
s->stksize = k;
GDKfree(old);
diff --git a/monetdb5/optimizer/opt_mergetable.c 
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -1647,8 +1647,9 @@ OPTmergetableImplementation(Client cntxt
ml.vsize = mb->vsize;
ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
-   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) 
+   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) {
goto cleanup;
+   }
for (i=0; i 0){
 chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c
--- a/monetdb5/optimizer/opt_reorder.c
+++ b/monetdb5/optimizer/opt_reorder.c
@@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr 
sz += list[i]->used;
}
uselist = GDKzalloc(sizeof(int)*sz);
+   if (!uselist) {
+   GDKfree(list);
+   GDKfree(var);
+   return NULL;
+   }
 
for(i=0;istop; i++) {
if (list[i]->cnt) {
diff --git a/sql/backends/monet5/sql_optimizer.c 
b/sql/backends/monet5/sql_optimizer.c
--- a/sql/backends/monet5/sql_optimizer.c
+++ b/sql/backends/monet5/sql_optimizer.c
@@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu
return size;
 }
 
-static lng 
+static

MonetDB: default - moar spaces

2017-01-06 Thread Hannes Muehleisen

Changeset: c37ce964753c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c37ce964753c
Modified Files:
sql/test/malloc_fail/Tests/mallocs.SQL.py
Branch: default
Log Message:

moar spaces


diffs (33 lines):

diff --git a/sql/test/malloc_fail/Tests/mallocs.SQL.py 
b/sql/test/malloc_fail/Tests/mallocs.SQL.py
--- a/sql/test/malloc_fail/Tests/mallocs.SQL.py
+++ b/sql/test/malloc_fail/Tests/mallocs.SQL.py
@@ -10,19 +10,19 @@ cursor = dbh.cursor()
 q = """
 
 create procedure setmallocsuccesscount(count BIGINT)
-   external name "io"."setmallocsuccesscount";
+external name "io"."setmallocsuccesscount";
 
 call setmallocsuccesscount(%d);
 SELECT * FROM tables;
 """
 i = 3300
 while i > 1000:
-   #print(i)
-   i-=1
-   try:
-   cursor.execute(q % (i))
-   except Exception as e:
-   #print(e)
-   pass
-   finally:
-   dbh.rollback()
+#print(i)
+i-=1
+try:
+cursor.execute(q % (i))
+except Exception as e:
+#print(e)
+pass
+finally:
+dbh.rollback()
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Merge with Dec2016

2017-01-06 Thread Hannes Muehleisen

Changeset: 7aee4214df1a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7aee4214df1a
Modified Files:
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_stack.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_reorder.c
sql/backends/monet5/sql_optimizer.c
Branch: default
Log Message:

Merge with Dec2016


diffs (141 lines):

diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c
--- a/monetdb5/mal/mal_builder.c
+++ b/monetdb5/mal/mal_builder.c
@@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val
}
cst.len= (int) strlen(cst.val.sval);
getArg(q,0) = defConstant(mb,TYPE_str,);
+   if (getArg(q,0) < 0) {
+   freeInstruction(q);
+   return NULL;
+   }
clrVarConstant(mb,getArg(q,0));
setVarDisabled(mb,getArg(q,0));
if (mb->errors) {
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -717,7 +717,6 @@ makeVarSpace(MalBlkPtr mb)
if (new == NULL) {
mb->errors++;
showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no 
storage left\n");
-   assert(0);
return -1;
}
memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr));
@@ -1245,6 +1244,9 @@ defConstant(MalBlkPtr mb, int type, ValP
return k;
}
k = newTmpVariable(mb, type);
+   if (k == -1) {
+   return k;
+   }
setVarConstant(mb, k);
setVarFixed(mb, k);
if (type >= 0 && type < GDKatomcnt && ATOMextern(type))
diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c
--- a/monetdb5/mal/mal_interpreter.c
+++ b/monetdb5/mal/mal_interpreter.c
@@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size)
 
assert(size >= mb->vsize);
stk = newGlobalStack(size);
+   if (!stk) {
+   return NULL;
+   }
//memset((char *)stk, 0, stackSize(size)); already set
//stk->stksize = size;
stk->stktop = mb->vtop;
diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c
--- a/monetdb5/mal/mal_stack.c
+++ b/monetdb5/mal/mal_stack.c
@@ -59,8 +59,9 @@ newGlobalStack(int size)
MalStkPtr s;
 
s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk));
-   if (s == NULL)
-   GDKfatal("newGlobalStack:can not obtain memory\n");
+   if (!s) {
+   return NULL;
+   }
s->stksize = size;
return s;
 }
@@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn
return old;
k = ((cnt / STACKINCR) + 1) * STACKINCR;
s = newGlobalStack(k);
+   if (!s) {
+   return NULL;
+   }
memcpy(s, old, stackSize(old->stksize));
s->stksize = k;
GDKfree(old);
diff --git a/monetdb5/optimizer/opt_mergetable.c 
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -1544,8 +1544,9 @@ OPTmergetableImplementation(Client cntxt
ml.vsize = mb->vsize;
ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
-   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) 
+   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) {
goto cleanup;
+   }
for (i=0; i 0){
 chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c
--- a/monetdb5/optimizer/opt_reorder.c
+++ b/monetdb5/optimizer/opt_reorder.c
@@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr 
sz += list[i]->used;
}
uselist = GDKzalloc(sizeof(int)*sz);
+   if (!uselist) {
+   GDKfree(list);
+   GDKfree(var);
+   return NULL;
+   }
 
for(i=0;istop; i++) {
if (list[i]->cnt) {
diff --git a/sql/backends/monet5/sql_optimizer.c 
b/sql/backends/monet5/sql_optimizer.c
--- a/sql/backends/monet5/sql_optimizer.c
+++ b/sql/backends/monet5/sql_optimizer.c
@@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu
return size;
 }
 
-static

MonetDB: default - new test case for malloc failures

2017-01-06 Thread Hannes Muehleisen

Changeset: 82dad11b6fd3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=82dad11b6fd3
Added Files:
sql/test/malloc_fail/Tests/mallocs.SQL.py
Removed Files:
sql/test/malloc_fail/Tests/initialize.sql
sql/test/malloc_fail/Tests/initialize.stable.err
sql/test/malloc_fail/Tests/initialize.stable.out
sql/test/malloc_fail/Tests/setmemorylimit-fail.reqtests
sql/test/malloc_fail/Tests/setmemorylimit-fail.sql
sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err
sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out
sql/test/malloc_fail/Tests/setmemorylimit-fail2.reqtests
sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql
sql/test/malloc_fail/Tests/setmemorylimit.reqtests
sql/test/malloc_fail/Tests/setmemorylimit.sql
sql/test/malloc_fail/Tests/setmemorylimit.stable.err
sql/test/malloc_fail/Tests/setmemorylimit.stable.out
Modified Files:
gdk/gdk_utils.c
gdk/gdk_utils.h
monetdb5/mal/mal_exception.c
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_io.h
monetdb5/modules/mal/mal_io.mal
sql/test/malloc_fail/Tests/All
Branch: default
Log Message:

new test case for malloc failures


diffs (truncated from 527 to 300 lines):

diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -319,7 +319,7 @@ int GDK_vm_trim = 1;
 #include "gdk_atomic.h"
 static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0;
 #ifndef NDEBUG
-static volatile lng GDK_mallocedbytes_limit = -1;
+static volatile lng GDK_malloc_success_count = -1;
 #endif
 static volatile ATOMIC_TYPE GDK_vm_cursize = 0;
 #ifdef GDK_VM_KEEPHISTO
@@ -1639,12 +1639,17 @@ GDKmalloc_prefixsize(size_t size)
return s;
 }
 
+#ifndef NDEBUG
+static MT_Lock mallocsuccesslock;
+#endif
+
 void
-GDKsetmemorylimit(lng nbytes)
+GDKsetmallocsuccesscount(lng count)
 {
-   (void) nbytes;
+   (void) count;
 #ifndef NDEBUG
-   GDK_mallocedbytes_limit = nbytes;
+   MT_lock_init(, "mallocsuccesslock");
+   GDK_malloc_success_count = count;
 #endif
 }
 
@@ -1668,10 +1673,15 @@ GDKmallocmax(size_t size, size_t *maxsiz
}
 #ifndef NDEBUG
/* fail malloc for testing purposes depending on set limit */
-   if (GDK_mallocedbytes_limit >= 0 &&
-   size > (size_t) GDK_mallocedbytes_limit) {
+   if (GDK_malloc_success_count > 0) {
+   MT_lock_set();
+   if (GDK_malloc_success_count > 0) GDK_malloc_success_count--;
+   MT_lock_unset();
+   }
+   if (GDK_malloc_success_count == 0) {
return NULL;
}
+
 #endif
size = (size + 7) & ~7; /* round up to a multiple of eight */
s = GDKmalloc_prefixsize(size);
diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h
--- a/gdk/gdk_utils.h
+++ b/gdk/gdk_utils.h
@@ -77,7 +77,7 @@ gdk_export void MT_init(void);/*  init 
 gdk_export int GDKinit(opt *set, int setlen);
 
 /* used for testing only */
-gdk_export void GDKsetmemorylimit(lng nbytes);
+gdk_export void GDKsetmallocsuccesscount(lng count);
 
 /*
  * Upon closing the session, all persistent BATs should be saved and
diff --git a/monetdb5/mal/mal_exception.c b/monetdb5/mal/mal_exception.c
--- a/monetdb5/mal/mal_exception.c
+++ b/monetdb5/mal/mal_exception.c
@@ -57,7 +57,10 @@ createExceptionInternal(enum malexceptio
 {
char *message;
int len;
-
+   // if there is an error we allow memory allocation once again
+#ifndef NDEBUG
+   GDKsetmallocsuccesscount(-1);
+#endif
message = GDKmalloc(GDKMAXERRLEN);
if (message == NULL)
return M5OutOfMemory;   /* last resort */
diff --git a/monetdb5/modules/mal/mal_io.c b/monetdb5/modules/mal/mal_io.c
--- a/monetdb5/modules/mal/mal_io.c
+++ b/monetdb5/modules/mal/mal_io.c
@@ -779,8 +779,8 @@ IOimport(void *ret, bat *bid, str *fnme)
 
 
 str
-IOsetmemorylimit(void *res, lng *nbytes) {
+IOsetmallocsuccesscount(void *res, lng *count) {
(void) res;
-   GDKsetmemorylimit(*nbytes);
+   GDKsetmallocsuccesscount(*count);
return MAL_SUCCEED;
 }
diff --git a/monetdb5/modules/mal/mal_io.h b/monetdb5/modules/mal/mal_io.h
--- a/monetdb5/modules/mal/mal_io.h
+++ b/monetdb5/modules/mal/mal_io.h
@@ -29,6 +29,6 @@ mal_export str io_stdin(Client cntxt, Ma
 mal_export str io_stdout(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
 mal_export str io_stderr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
 
-mal_export str IOsetmemorylimit(void *res, lng *nbytes);
+mal_export str IOsetmallocsuccesscount(void *res, lng *nbytes);
 
 #endif /* _PRINT_H_ */
diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal
--- a/monetdb5/modules/mal/mal_io.mal
+++ b/monetdb5/modules/mal/mal_io.mal
@@ -62,6 +62,6 @@ comment "Import a BAT from an ASCII dump
  else parsing errors will occur as an exception.";
 
 # used for

MonetDB: Dec2016 - Check for malloc failures in several places

2017-01-06 Thread Hannes Muehleisen

Changeset: 039e1a5a4266 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=039e1a5a4266
Modified Files:
monetdb5/mal/mal_builder.c
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_stack.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_reorder.c
sql/backends/monet5/sql_optimizer.c
Branch: Dec2016
Log Message:

Check for malloc failures in several places


diffs (141 lines):

diff --git a/monetdb5/mal/mal_builder.c b/monetdb5/mal/mal_builder.c
--- a/monetdb5/mal/mal_builder.c
+++ b/monetdb5/mal/mal_builder.c
@@ -108,6 +108,10 @@ newComment(MalBlkPtr mb, const char *val
}
cst.len= (int) strlen(cst.val.sval);
getArg(q,0) = defConstant(mb,TYPE_str,);
+   if (getArg(q,0) < 0) {
+   freeInstruction(q);
+   return NULL;
+   }
clrVarConstant(mb,getArg(q,0));
setVarDisabled(mb,getArg(q,0));
if (mb->errors) {
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -730,7 +730,6 @@ makeVarSpace(MalBlkPtr mb)
if (new == NULL) {
mb->errors++;
showScriptException(GDKout, mb, 0, MAL, "newMalBlk:no 
storage left\n");
-   assert(0);
return -1;
}
memset(new + mb->vsize, 0, (s - mb->vsize) * sizeof(VarPtr));
@@ -1258,6 +1257,9 @@ defConstant(MalBlkPtr mb, int type, ValP
return k;
}
k = newTmpVariable(mb, type);
+   if (k == -1) {
+   return k;
+   }
setVarConstant(mb, k);
setVarFixed(mb, k);
if (type >= 0 && type < GDKatomcnt && ATOMextern(type))
diff --git a/monetdb5/mal/mal_interpreter.c b/monetdb5/mal/mal_interpreter.c
--- a/monetdb5/mal/mal_interpreter.c
+++ b/monetdb5/mal/mal_interpreter.c
@@ -285,6 +285,9 @@ prepareMALstack(MalBlkPtr mb, int size)
 
assert(size >= mb->vsize);
stk = newGlobalStack(size);
+   if (!stk) {
+   return NULL;
+   }
//memset((char *)stk, 0, stackSize(size)); already set
//stk->stksize = size;
stk->stktop = mb->vtop;
diff --git a/monetdb5/mal/mal_stack.c b/monetdb5/mal/mal_stack.c
--- a/monetdb5/mal/mal_stack.c
+++ b/monetdb5/mal/mal_stack.c
@@ -59,8 +59,9 @@ newGlobalStack(int size)
MalStkPtr s;
 
s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk));
-   if (s == NULL)
-   GDKfatal("newGlobalStack:can not obtain memory\n");
+   if (!s) {
+   return NULL;
+   }
s->stksize = size;
return s;
 }
@@ -75,6 +76,9 @@ reallocGlobalStack(MalStkPtr old, int cn
return old;
k = ((cnt / STACKINCR) + 1) * STACKINCR;
s = newGlobalStack(k);
+   if (!s) {
+   return NULL;
+   }
memcpy(s, old, stackSize(old->stksize));
s->stksize = k;
GDKfree(old);
diff --git a/monetdb5/optimizer/opt_mergetable.c 
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -1647,8 +1647,9 @@ OPTmergetableImplementation(Client cntxt
ml.vsize = mb->vsize;
ml.horigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
ml.torigin = (int*) GDKmalloc(sizeof(int)* ml.vsize);
-   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) 
+   if ( ml.v == NULL || ml.horigin == NULL || ml.torigin == NULL) {
goto cleanup;
+   }
for (i=0; i 0){
 chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
diff --git a/monetdb5/optimizer/opt_reorder.c b/monetdb5/optimizer/opt_reorder.c
--- a/monetdb5/optimizer/opt_reorder.c
+++ b/monetdb5/optimizer/opt_reorder.c
@@ -134,6 +134,11 @@ OPTdependencies(Client cntxt, MalBlkPtr 
sz += list[i]->used;
}
uselist = GDKzalloc(sizeof(int)*sz);
+   if (!uselist) {
+   GDKfree(list);
+   GDKfree(var);
+   return NULL;
+   }
 
for(i=0;istop; i++) {
if (list[i]->cnt) {
diff --git a/sql/backends/monet5/sql_optimizer.c 
b/sql/backends/monet5/sql_optimizer.c
--- a/sql/backends/monet5/sql_optimizer.c
+++ b/sql/backends/monet5/sql_optimizer.c
@@ -60,11 +60,11 @@ SQLgetColumnSize(sql_trans *tr, sql_colu

MonetDB: Dec2016 - Allow aggregation count optimization in more ...

2017-01-03 Thread Hannes Muehleisen

Changeset: bf160289ab50 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bf160289ab50
Modified Files:
gdk/gdk_aggr.c
Branch: Dec2016
Log Message:

Allow aggregation count optimization in more cases on Sjoerd's request


diffs (12 lines):

diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -1386,7 +1386,7 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT 
return GDK_SUCCEED;
}
 
-   if (!skip_nils && (e == NULL ||
+   if ((!skip_nils || cntsp == NULL || b->tnonil) && (e == NULL ||
 (BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) &&
(BATtdense(g) || (g->tkey && g->tnonil))) {
/* trivial: singleton groups, so all results are equal
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - Fix for Bug 6178

2017-01-03 Thread Hannes Muehleisen

Changeset: c10ab2216eec for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c10ab2216eec
Modified Files:
gdk/gdk_aggr.c
monetdb5/optimizer/opt_mergetable.c
Branch: Dec2016
Log Message:

Fix for Bug 6178


diffs (27 lines):

diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -1386,7 +1386,7 @@ BATgroupavg(BAT **bnp, BAT **cntsp, BAT 
return GDK_SUCCEED;
}
 
-   if ((e == NULL ||
+   if (!skip_nils && (e == NULL ||
 (BATcount(e) == BATcount(b) && e->hseqbase == b->hseqbase)) &&
(BATtdense(g) || (g->tkey && g->tnonil))) {
/* trivial: singleton groups, so all results are equal
diff --git a/monetdb5/optimizer/opt_mergetable.c 
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -1107,10 +1107,7 @@ mat_group_aggr(MalBlkPtr mb, InstrPtr p,
ai2 = pushArgument(mb, ai2, getArg(ai1, 0));
ai2 = pushArgument(mb, ai2, mat[g].mv);
ai2 = pushArgument(mb, ai2, mat[e].mv);
-   if (isAvg)
-   ai2 = pushBit(mb, ai2, 0); /* do not skip nils */
-   else
-   ai2 = pushBit(mb, ai2, 1); /* skip nils */
+   ai2 = pushBit(mb, ai2, 1); /* skip nils */
if (getFunctionId(p) != subminRef && getFunctionId(p) != submaxRef)
ai2 = pushBit(mb, ai2, 1);
pushInstruction(mb, ai2);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - Update for Bug 6178, issue only appears when ...

2017-01-03 Thread Hannes Muehleisen

Changeset: a9ebbd5b2e53 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a9ebbd5b2e53
Modified Files:
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
Branch: Dec2016
Log Message:

Update for Bug 6178, issue only appears when using AVG


diffs (92 lines):

diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql 
b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
--- a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
+++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
@@ -6,14 +6,22 @@ COPY 2 RECORDS INTO x FROM STDIN USING D
 "N907MQ",
 "N907MQ",191
 
-SELECT * FROM x;
+-- correct result, 191
+SELECT AVG( arr_delay ) FROM x;
+
+-- wrong result, NULL
 SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum;
 
--- these two should be the same?!
--- N907MQ|NULL is in the data, hence N907MQ|NULL should also be the result
+-- works fine with MIN
+SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum;
 
+-- correct again
 SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY 
tailnum;
+
+-- wrong again
 SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) AS 
xxx WHERE tailnum = 'N907MQ';
 
+-- both work fine with MIN
+SELECT tailnum , MIN( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY 
tailnum;
+SELECT * FROM ( SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum ) AS 
xxx WHERE tailnum = 'N907MQ';
 
-
diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out 
b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
--- a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
+++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
@@ -64,31 +64,48 @@ Ready.
 #"N907MQ",
 #"N907MQ",191
 [ 2]
-#SELECT * FROM x;
-% sys.x,   sys.x # table_name
-% tailnum, arr_delay # name
-% clob,int # type
-% 6,   3 # length
-[ "N907MQ",NULL]
-[ "N907MQ",191 ]
+#SELECT AVG( arr_delay ) FROM x;
+% sys.L4 # table_name
+% L3 # name
+% double # type
+% 24 # length
+[ 191  ]
 #SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum;
 % sys.x,   sys.L4 # table_name
 % tailnum, L3 # name
 % clob,double # type
 % 6,   24 # length
-[ "N907MQ",NULL]
+[ "N907MQ",191 ]
+#SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum;
+% sys.x,   sys.L4 # table_name
+% tailnum, L3 # name
+% clob,int # type
+% 6,   3 # length
+[ "N907MQ",191 ]
 #SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY 
tailnum;
 % sys.x,   sys.L4 # table_name
 % tailnum, L3 # name
 % clob,double # type
 % 6,   24 # length
-[ "N907MQ",NULL]
+[ "N907MQ",191 ]
 #SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) 
AS xxx WHERE tailnum = 'N907MQ';
 % sys.xxx, sys.xxx # table_name
 % tailnum, L3 # name
 % clob,double # type
 % 6,   24 # length
-[ "N907MQ",NULL]
+[ "N907MQ",191 ]
+#SELECT tailnum , MIN( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY 
tailnum;
+% sys.x,   sys.L4 # table_name
+% tailnum, L3 # name
+% clob,int # type
+% 6,   3 # length
+[ "N907MQ",191 ]
+#SELECT * FROM ( SELECT tailnum , MIN( arr_delay ) FROM x GROUP BY tailnum ) 
AS xxx WHERE tailnum = 'N907MQ';
+% sys.xxx, sys.xxx # table_name
+% tailnum, L3 # name
+% clob,int # type
+% 6,   3 # length
+[ "N907MQ",191 ]
 
 # 18:20:39 >  
 # 18:20:39 >  "Done."
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Dec2016 - Smaller test for Bug 6178

2017-01-02 Thread Hannes Muehleisen

Changeset: 41e0ace65aed for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=41e0ace65aed
Added Files:
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err
sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
Modified Files:
sql/test/BugTracker-2017/Tests/All
Branch: Dec2016
Log Message:

Smaller test for Bug 6178


diffs (171 lines):

diff --git a/sql/test/BugTracker-2017/Tests/All 
b/sql/test/BugTracker-2017/Tests/All
--- a/sql/test/BugTracker-2017/Tests/All
+++ b/sql/test/BugTracker-2017/Tests/All
@@ -1,1 +1,2 @@
 shutdown.Bug-6182
+avggroupbysq.Bug-6178
diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql 
b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.sql
@@ -0,0 +1,19 @@
+START TRANSACTION;
+
+CREATE TABLE x (tailnum STRING, arr_delay INTEGER);
+
+COPY 2 RECORDS INTO x FROM STDIN USING DELIMITERS ',','\n','"' NULL as '';
+"N907MQ",
+"N907MQ",191
+
+SELECT * FROM x;
+SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum;
+
+-- these two should be the same?!
+-- N907MQ|NULL is in the data, hence N907MQ|NULL should also be the result
+
+SELECT tailnum , AVG( arr_delay ) FROM x WHERE tailnum = 'N907MQ' GROUP BY 
tailnum;
+SELECT * FROM ( SELECT tailnum , AVG( arr_delay ) FROM x GROUP BY tailnum ) AS 
xxx WHERE tailnum = 'N907MQ';
+
+
+
diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err 
b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'avggroupbysq.Bug-6178` in directory 'sql/test/BugTracker-2017` 
itself:
+
+
+# 18:20:38 >  
+# 18:20:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=36572" "--set" 
"mapi_usock=/var/tmp/mtest-8698/.s.monetdb.36572" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017"
 "--set" "embedded_py=true"
+# 18:20:38 >  
+
+# builtin opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 36572
+# cmdline opt  mapi_usock = /var/tmp/mtest-8698/.s.monetdb.36572
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017
+# cmdline opt  embedded_py = true
+# cmdline opt  gdk_debug = 536870922
+
+# 18:20:38 >  
+# 18:20:38 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-8698" "--port=36572"
+# 18:20:38 >  
+
+
+# 18:20:39 >  
+# 18:20:39 >  "Done."
+# 18:20:39 >  
+
diff --git a/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out 
b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2017/Tests/avggroupbysq.Bug-6178.stable.out
@@ -0,0 +1,96 @@
+stdout of test 'avggroupbysq.Bug-6178` in directory 'sql/test/BugTracker-2017` 
itself:
+
+
+# 18:20:38 >  
+# 18:20:38 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=36572" "--set" 
"mapi_usock=/var/tmp/mtest-8698/.s.monetdb.36572" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_BugTracker-2017"
 "--set" "embedded_py=true"
+# 18:20:38 >  
+
+# MonetDB 5 server v11.25.4
+# This is an unreleased version
+# Serving database 'mTests_sql_test_BugTracker-2017', using 4 threads
+# Compiled for x86_64-apple-darwin16.3.0/64bit with 128bit integers
+# Found 16.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2017 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36572/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-8698/.s.monetdb.36572
+# MonetDB/SQL module loaded
+
+
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql

MonetDB: queryid - added query id export to monet

2016-12-05 Thread Hannes Muehleisen

Changeset: 0c4024427f78 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c4024427f78
Modified Files:
sql/backends/monet5/mal_backend.h
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
Branch: queryid
Log Message:

added query id export to monet


diffs (75 lines):

diff --git a/sql/backends/monet5/mal_backend.h 
b/sql/backends/monet5/mal_backend.h
--- a/sql/backends/monet5/mal_backend.h
+++ b/sql/backends/monet5/mal_backend.h
@@ -38,7 +38,7 @@ typedef struct backend {
stream  *out;
ofmtoutput_format;  /* csv, json */
Client  client;
-   MalBlkPtr mb;   /* needed during mal generation */
+   MalBlkPtr mb;   /* needed during mal generation and result set 
export */
int mvc_var;
int vtop;   /* top of the variable stack before the current 
function */
cq  *q; /* pointer to the cached query */
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -2825,6 +2825,9 @@ mvc_result_set_wrap( Client cntxt, MalBl
BBPunfix(bid);
}
/* now sent it to the channel cntxt->fdout */
+
+   ((backend*)(cntxt->sqlcontext))->mb = mb;
+
if (mvc_export_result(cntxt->sqlcontext, cntxt->fdout, res))
msg = createException(SQL, "sql.resultset", "failed");
   wrapup_result_set:
@@ -3290,6 +3293,9 @@ mvc_affected_rows_wrap(Client cntxt, Mal
assert(mtype == TYPE_lng);
nr = *getArgReference_lng(stk, pci, 2);
b = cntxt->sqlcontext;
+
+   b->mb = mb; // to allow query id export in result set
+
error = mvc_export_affrows(b, b->out, nr, "");
if (error)
throw(SQL, "sql.affectedRows", "failed");
@@ -3401,6 +3407,8 @@ mvc_scalar_value_wrap(Client cntxt, MalB
if (ATOMextern(mtype))
p = *(ptr *) p;
 
+   b->mb = mb; // for query id export
+
// scalar values are single-column result sets
mvc_result_table(b->mvc, 1, 1, NULL);
mvc_result_value(b->mvc, *tn, *cn, *type, *digits, *scale, p, mtype);
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -1610,7 +1610,7 @@ mvc_export_value(backend *b, stream *s, 
assert(qtype == Q_TABLE);
 
if (csv && 
-  (mnstr_write(s, "&1 0 1 1 1\n", 11, 1) != 1 ||
+  (mnstr_printf(s, "&1 0 1 1 1 "OIDFMT"\n", b->mb->tag) != 1 ||
/* fallback to default tuplecount (1) and id (0) */
/* TODO first header name then values */
mnstr_write(s, "% ", 2, 1) != 1 || 
@@ -1681,7 +1681,7 @@ mvc_export_affrows(backend *b, stream *s
 
m->rowcnt = val;
stack_set_number(m, "rowcnt", m->rowcnt);
-   if (mnstr_write(s, "&2 ", 3, 1) != 1 || !mvc_send_lng(s, val) || 
mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, m->last_id) || 
mnstr_write(s, "\n", 1, 1) != 1)
+   if (mnstr_write(s, "&2 ", 3, 1) != 1 || !mvc_send_lng(s, val) || 
mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, m->last_id) || 
mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, (lng) b->mb->tag) || 
mnstr_write(s, "\n", 1, 1) != 1)
return -1;
if (mvc_export_warning(s, w) != 1)
return -1;
@@ -1741,6 +1741,9 @@ mvc_export_head(backend *b, stream *s, i
if (!mvc_send_int(s, (m->reply_size >= 0 && (BUN) m->reply_size < 
count) ? m->reply_size : (int) count))
return -1;
 
+   if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, (lng) 
b->mb->tag))
+   return -1;
+
if (mnstr_write(s, "\n% ", 3, 1) != 1)
return -1;
for (i = 0; i < t->nr_cols; i++) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - its been several years since clang liked the ...

2016-11-30 Thread Hannes Muehleisen

Changeset: d432f9c891fb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d432f9c891fb
Modified Files:
configure.ag
Branch: default
Log Message:

its been several years since clang liked the -pthread flag


diffs (12 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1450,7 +1450,7 @@ AS_VAR_IF([have_pthread], [no], [], [
save_LIBS="$LIBS"
save_CPPFLAGS="$CPPFLAGS"
AS_CASE([$GCC-$have_pthread-$CC_ver],
-   
[yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*|yes-auto-clang-8.*|yes-yes-clang-8.*],
 [
+   [yes-auto-clang-*|yes-yes-clang-*], [
# clang 5.*/6.*/7.* (Xcode 6.0) does not
# seem to have / require -pthread as compiler
# option; on Mac OS X Yosamite, "Apple LLVM
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - r tests using mfilter variable output now

2016-10-25 Thread Hannes Muehleisen

Changeset: 5d432332635f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d432332635f
Modified Files:
clients/R/Tests/dbi.stable.err
clients/R/Tests/deps-install.R
clients/R/Tests/deps-test.R
clients/R/Tests/install.R
Branch: default
Log Message:

r tests using mfilter variable output now


diffs (85 lines):

diff --git a/clients/R/Tests/dbi.stable.err b/clients/R/Tests/dbi.stable.err
--- a/clients/R/Tests/dbi.stable.err
+++ b/clients/R/Tests/dbi.stable.err
@@ -37,6 +37,7 @@ Use 'dbIsValid' instead.
 See help("Deprecated") 
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
+Identifier(s) "Need to quote this table name" contain uppercase or reserved 
SQL characters and need(s) to be quoted in queries.
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
 Treating character vector parameter as file name(s) for monetdb.read.csv()
 Identifier(s) "Sepal.Length", "Sepal.Width", "Petal.Length", "Petal.Width", 
"Species" contain uppercase or reserved SQL characters and need(s) to be quoted 
in queries.
diff --git a/clients/R/Tests/deps-install.R b/clients/R/Tests/deps-install.R
--- a/clients/R/Tests/deps-install.R
+++ b/clients/R/Tests/deps-install.R
@@ -1,8 +1,15 @@
 # autoinstall DBI and digest, we need those to install MonetDB.R
-dd <- capture.output(suppressMessages(suppressWarnings({
-   repos <- "http://cran.rstudio.com/;
-   lp <- c("devtools", "digest")
-   install.packages(lp, repos=repos, quiet=T)
-   update.packages(repos=repos, ask=F, oldPkgs=lp)
-   devtools::install_github("rstats-db/DBI", quiet=T)
-})))
+
+cat("#~BeginProfilingOutput~#\n", file=stderr())
+cat("#~BeginProfilingOutput~#\n", file=stdout())
+
+repos <- "http://cran.rstudio.com/;
+packages_required <- c("devtools", "digest")
+install.packages(packages_required, repos=repos, quiet=T)
+update.packages(repos=repos, ask=F, oldPkgs=packages_required)
+devtools::install_github("rstats-db/DBI", quiet=T)
+
+cat("#~EndProfilingOutput~#\n", file=stderr())
+cat("#~EndProfilingOutput~#\n\n", file=stdout())
+
+stopifnot(all(c(packages_required, "DBI") %in% 
installed.packages()[,"Package"]))
diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R
--- a/clients/R/Tests/deps-test.R
+++ b/clients/R/Tests/deps-test.R
@@ -1,8 +1,21 @@
-dd <- capture.output(suppressMessages(suppressWarnings({
-   (function(lp) {
+packages_required <- c('Rcpp', 'dplyr', 'Lahman', 'nycflights13')
+
+install_or_upgrade_packages <- function(lp) {
np <- lp[!(lp %in% installed.packages()[,"Package"])]
repos <- 'http://cran.rstudio.com/'
if(length(np)) install.packages(np, repos=repos, quiet=T)
update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T)
-   })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata'))
-})))
+}
+
+
+cat("#~BeginProfilingOutput~#\n", file=stderr())
+cat("#~BeginProfilingOutput~#\n", file=stdout())
+
+devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T)
+
+install_or_upgrade_packages(packages_required)
+
+cat("#~EndProfilingOutput~#\n", file=stderr())
+cat("#~EndProfilingOutput~#\n", file=stdout())
+
+stopifnot(all(packages_required %in% installed.packages()[,"Package"]))
diff --git a/clients/R/Tests/install.R b/clients/R/Tests/install.R
--- a/clients/R/Tests/install.R
+++ b/clients/R/Tests/install.R
@@ -1,4 +1,11 @@
-dd <- capture.output(suppressMessages( {
-   devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T)
-}))
-library(MonetDBLite,quietly=T)
+cat("#~BeginProfilingOutput~#\n", file=stderr())
+cat("#~BeginProfilingOutput~#\n", file=stdout())
+
+devtools::install_github("hannesmuehleisen/MonetDBLite", quiet=T)
+
+cat("#~EndProfilingOutput~#\n", file=stderr())
+cat("#~EndProfilingOutput~#\n", file=stdout())
+
+stopifnot("MonetDBLite" %in% installed.packages()[,"Package"])
+
+library(MonetDBLite, quietly=T)
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: k3match-jun2016 - fixes for compilation, tests worka ga...

2016-09-29 Thread Hannes Muehleisen

Changeset: 8c39d7692fbb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8c39d7692fbb
Modified Files:
sql/backends/monet5/Makefile.ag
sql/backends/monet5/k3m/k3m.c
Branch: k3match-jun2016
Log Message:

fixes for compilation, tests worka gain, praise something


diffs (51 lines):

diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag
--- a/sql/backends/monet5/Makefile.ag
+++ b/sql/backends/monet5/Makefile.ag
@@ -4,7 +4,7 @@
 #
 # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
 
-SUBDIRS = NOT_WIN32?vaults UDF LSST HAVE_GSL?gsl generator
+SUBDIRS = NOT_WIN32?vaults UDF LSST HAVE_GSL?gsl generator HAVE_K3M?k3m
 
 INCLUDES = ../../include ../../common ../../storage ../../server \
   ../../../monetdb5/modules/atoms \
diff --git a/sql/backends/monet5/k3m/k3m.c b/sql/backends/monet5/k3m/k3m.c
--- a/sql/backends/monet5/k3m/k3m.c
+++ b/sql/backends/monet5/k3m/k3m.c
@@ -63,7 +63,7 @@ str K3Mbuild(Client cntxt, MalBlkPtr mb,
bit newtree = !k3m_tree;
k3m_tree_tpe *k3m_tree_alloc = NULL;
(void) cntxt;
-   MT_lock_set(_lock, "K3Mbuild");
+   MT_lock_set(_lock);
 
if (!isaBatType(getArgType(mb,pci,0)) || 
!isaBatType(getArgType(mb,pci,1)) ||
!isaBatType(getArgType(mb,pci,2)) || 
!isaBatType(getArgType(mb,pci,3))) {
@@ -158,7 +158,7 @@ str K3Mbuild(Client cntxt, MalBlkPtr mb,
k3m_tree->tree = k3m_insert_node(k3m_tree->tree, 
_tree_alloc->tree[i]);
}
}
-   MT_lock_unset(_lock, "K3Mbuild");
+   MT_lock_unset(_lock);
ret = BATnew(TYPE_void, TYPE_bit, 0, TRANSIENT);
BUNappend(ret, , 0);
*getArgReference_bat(stk, pci, 0) = ret->batCacheid;
@@ -179,7 +179,7 @@ str K3Mfree(Client cntxt, MalBlkPtr mb, 
(void) stk;
(void) pci;
 
-   MT_lock_set(_lock, "K3Mfree");
+   MT_lock_set(_lock);
 
for (i = 0; i < k3m_allocs_pos; i++) {
GDKfree(k3m_allocs[i]->tree);
@@ -192,7 +192,7 @@ str K3Mfree(Client cntxt, MalBlkPtr mb, 
k3m_allocs_size = K3M_ALLOCS_DEFAULT_SIZE;
k3m_tree = NULL;
 
-   MT_lock_unset(_lock, "K3Mfree");
+   MT_lock_unset(_lock);
 
ret = BATnew(TYPE_void, TYPE_bit, 0, TRANSIENT);
BUNappend(ret, , 0);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: k3match-jun2016 - new branch

2016-09-29 Thread Hannes Muehleisen

Changeset: 7b8a09513771 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7b8a09513771
Added Files:
sql/backends/monet5/k3m/3dtree.c
sql/backends/monet5/k3m/3dtree.h
sql/backends/monet5/k3m/67_k3m.mal
sql/backends/monet5/k3m/67_k3m.sql
sql/backends/monet5/k3m/Makefile.ag
sql/backends/monet5/k3m/Tests/All
sql/backends/monet5/k3m/Tests/k3m.inserttree.sql
sql/backends/monet5/k3m/Tests/k3m.inserttree.stable.err
sql/backends/monet5/k3m/Tests/k3m.inserttree.stable.out
sql/backends/monet5/k3m/Tests/k3m.singletable.sql
sql/backends/monet5/k3m/Tests/k3m.singletable.stable.err
sql/backends/monet5/k3m/Tests/k3m.singletable.stable.out
sql/backends/monet5/k3m/k3m.c
sql/backends/monet5/k3m/k3m.mal
sql/backends/monet5/k3m/k3match.h
sql/backends/monet5/k3m/median.c
sql/backends/monet5/k3m/median.h
sql/backends/monet5/k3m/point.c
sql/backends/monet5/k3m/point.h
Modified Files:
configure.ag
Branch: k3match-jun2016
Log Message:

new branch


diffs (truncated from 2529 to 300 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1877,7 +1877,7 @@ if test "x$have_pthread" != xno; then
save_LIBS="$LIBS"
save_CPPFLAGS="$CPPFLAGS"
case $GCC-$have_pthread-$CC_ver in
-   
yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*)
+   
yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*|yes-auto-clang-8.*|yes-yes-clang-8.*)
# clang 5.*/6.*/7.* (Xcode 6.0) does not
# seem to have / require -pthread as compiler
# option; on Mac OS X Yosamite, "Apple LLVM
@@ -2254,6 +2254,17 @@ if test "x$have_valgrind" != xno; then
[if test "x$have_valgrind" = xyes; then AC_MSG_ERROR([no 
valgrind support found]); fi])
 fi
 
+org_have_k3m=no
+have_k3m=$org_have_k3m
+AC_ARG_WITH(k3m,
+   AS_HELP_STRING([--with-k3m],
+   [include Pim Schellart's K3Match library (default=no)]),
+   have_k3m=$withval)
+if test "x$have_k3m" != xno; then
+   AC_DEFINE(HAVE_K3M, 1, [Define if you want to use the K3Match library])
+fi
+AM_CONDITIONAL(HAVE_K3M, test x"$have_k3m" != xno)
+
 # check for sphinxclient
 org_have_sphinxclient="auto"
 have_sphinxclient=$org_have_sphinxclient
@@ -3107,7 +3118,7 @@ AC_DEFINE([__hidden], [/* empty */],
 
 dnl checks for library functions
 case $host in
-   *-darwin1[[012345]]*)
+   *-darwin1[[0123456]]*)
# OSX 10.6 (Snow Leopard) and up somehow makes configure believe
# that fdatasync exists, in reality however, it does not on this
# platform.
@@ -3645,6 +3656,7 @@ for comp in \
'java ' \
'java_control ' \
'java_jdbc' \
+   'k3m  ' \
'liblas   ' \
'liblzma  ' \
'libxml2  ' \
diff --git a/sql/backends/monet5/k3m/3dtree.c b/sql/backends/monet5/k3m/3dtree.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/k3m/3dtree.c
@@ -0,0 +1,255 @@
+/**
+ * This file is part of K3Match.  *
+ * Copyright (C) 2016 Pim Schellart  *
+ **
+ * This Source Code Form is subject to the terms of the Mozilla Public*
+ * License, v. 2.0. If a copy of the MPL was not distributed with this*
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.   *
+ **/
+
+#include 
+#include 
+
+#include 
+
+#define SQUARE(x) ((x) * (x))
+
+void k3m_build_balanced_tree(node_t *tree, point_t **points, int_t npoints, 
int axis, int_t *npool)
+{
+  node_t *current = tree+(*npool);
+
+  int next_axis = (axis + 1) % 3;
+
+  int_t nleft, nright;
+
+  current->left = NULL;
+  current->right = NULL;
+  current->axis = axis;
+
+  current->point = k3m_median(points, npoints, current->axis);
+
+  nright = npoints / 2;
+  nleft = nright - (1 - npoints % 2);
+
+  if (nleft > 0)
+  {
+(*npool)++;
+current->left = tree+(*npool);
+current->left->parent = &(*current);
+k3m_build_balanced_tree(tree, points, nleft, next_axis, npool);
+  }
+
+  if (nright > 0)
+  {
+(*npool)++;
+current->right = tree+(*npool);
+current->right->parent = &(*current);
+k3m_build_balanced_tree(tree, points+nleft+1, nright, next_axis, npool);
+  }
+}
+
+node_t* k3m_insert_node(node_t *tree, node_t *node)
+{
+int axis = 0;
+node_t *parent = NULL;
+node_t *current = tree;
+
+node->left = NULL;
+node->right = NULL;
+

MonetDB: Jun2016 - fixed compilation for clang 8y.x

2016-09-26 Thread Hannes Muehleisen

Changeset: a11cf508e923 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a11cf508e923
Modified Files:
configure.ag
Branch: Jun2016
Log Message:

fixed compilation for clang 8y.x
(grafted from accc14b26ef358c46134fbbb675a69f5216920fd)


diffs (12 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1886,7 +1886,7 @@ if test "x$have_pthread" != xno; then
save_LIBS="$LIBS"
save_CPPFLAGS="$CPPFLAGS"
case $GCC-$have_pthread-$CC_ver in
-   
yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*)
+   
yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*|yes-auto-clang-8.*|yes-yes-clang-8.*)
# clang 5.*/6.*/7.* (Xcode 6.0) does not
# seem to have / require -pthread as compiler
# option; on Mac OS X Yosamite, "Apple LLVM
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: Jun2016 - fix compilation on osx 10.12

2016-09-23 Thread Hannes Muehleisen

Changeset: c299b5dca70d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c299b5dca70d
Modified Files:
configure.ag
Branch: Jun2016
Log Message:

fix compilation on osx 10.12


diffs (12 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -3116,7 +3116,7 @@ AC_DEFINE([__hidden], [/* empty */],
 
 dnl checks for library functions
 case $host in
-   *-darwin1[[012345]]*)
+   *-darwin1[[0123456]]*)
# OSX 10.6 (Snow Leopard) and up somehow makes configure believe
# that fdatasync exists, in reality however, it does not on this
# platform.
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - fix compilation on osx 10.12

2016-09-21 Thread Hannes Muehleisen

Changeset: 696d9567db67 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=696d9567db67
Modified Files:
configure.ag
Branch: default
Log Message:

fix compilation on osx 10.12


diffs (12 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2510,7 +2510,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 
 #checks for library functions
 AS_CASE([$host],
-   [*-darwin1[[012345]]*], [
+   [*-darwin1[[0123456]]*], [
# OSX 10.6 (Snow Leopard) and up somehow makes
# configure believe that fdatasync exists, in
# reality however, it does not on this platform.
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - fixed compilation for clang 8y.x

2016-09-20 Thread Hannes Muehleisen

Changeset: accc14b26ef3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=accc14b26ef3
Modified Files:
configure.ag
Branch: default
Log Message:

fixed compilation for clang 8y.x


diffs (12 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1603,7 +1603,7 @@ AS_VAR_IF([have_pthread], [no], [], [
save_LIBS="$LIBS"
save_CPPFLAGS="$CPPFLAGS"
AS_CASE([$GCC-$have_pthread-$CC_ver],
-   
[yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*],
 [
+   
[yes-auto-clang-5.*|yes-yes-clang-5.*|yes-auto-clang-6.*|yes-yes-clang-6.*|yes-auto-clang-7.*|yes-yes-clang-7.*|yes-auto-clang-8.*|yes-yes-clang-8.*],
 [
# clang 5.*/6.*/7.* (Xcode 6.0) does not
# seem to have / require -pthread as compiler
# option; on Mac OS X Yosamite, "Apple LLVM
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - fixed freeing results

2016-08-15 Thread Hannes Muehleisen

Changeset: a257b3e35ebc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a257b3e35ebc
Modified Files:
clients/mapilib/mapi.c
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

fixed freeing results


diffs (28 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -5605,7 +5605,7 @@ mapi_fetch_row(MapiHdl hdl)
result->tuple_count += res->row_count;
result->rows_read++;
if(hdl->mid->protobuf_res) {
-   
mhapi__query_result__free_unpacked(NULL, hdl->mid->protobuf_res);
+   
mhapi__query_result__free_unpacked(hdl->mid->protobuf_res, NULL);
}
hdl->mid->protobuf_res = (void*) res;
return result->fieldcnt;
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -2150,7 +2150,11 @@ static int mvc_export_resultset_prot10(r
assert(mhapi__query_result__get_packed_size() <= 
bsize);
assert(bs2_buffer(s).pos == 0);
bs2_setpos(s, mhapi__query_result__pack(, 
(uint8_t*) bs2_buffer(s).buf));
-   mnstr_flush(s);
+   if (mnstr_flush(s) < 0) {
+   fprintf(stderr, "Failed to flush.\n");
+   fres = -1;
+   goto cleanup;
+   }
 // TODO: free columns
srow = row;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - protbuf working

2016-08-15 Thread Hannes Muehleisen

Changeset: 8d84ca18bc45 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8d84ca18bc45
Added Files:
common/stream/mhapi.c
common/stream/mhapi.h
Modified Files:
bootstrap
clients/mapilib/Makefile.ag
clients/mapilib/mapi.c
common/stream/Makefile.ag
common/stream/stream.c
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

protbuf working


diffs (truncated from 590 to 300 lines):

diff --git a/bootstrap b/bootstrap
--- a/bootstrap
+++ b/bootstrap
@@ -61,4 +61,4 @@ if [ -f buildtools/conf/install-sh -a ! 
 fi
 
 
-protoc-c common/stream/mhapi.proto --c_out .
+# protoc-c common/stream/mhapi.proto --c_out .
diff --git a/clients/mapilib/Makefile.ag b/clients/mapilib/Makefile.ag
--- a/clients/mapilib/Makefile.ag
+++ b/clients/mapilib/Makefile.ag
@@ -13,7 +13,7 @@ lib_mapi = {
VERSION = $(MAPI_VERSION)
SOURCES = mapi.c mapi.rc
LIBS = $(SOCKET_LIBS) ../../common/stream/libstream \
-   ../../common/stream/libstream_protobuf \
+   ../../common/stream/libprotobuf \
../../common/options/libmoptions \
../../common/utils/libmcrypt $(openssl_LIBS) $(pfor_LIBS) 
$(protobuf_LIBS)
 }
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -5534,7 +5534,7 @@ mapi_split_line(MapiHdl hdl)
 }
 
 #ifdef HAVE_LIBPROTOBUF
-#include 
+#include 
 #endif
 
 int
@@ -5577,11 +5577,16 @@ mapi_fetch_row(MapiHdl hdl)
 
 
if (hdl->mid->colcomp == COLUMN_COMPRESSION_PROTOBUF) {
-   buffer buf = bs2_buffer(hdl->mid->from);
+   char dummy;
+   buffer buf;
 #ifndef HAVE_LIBPROTOBUF
// TODO: complain
 #else
-   Mhapi__QueryResult *res = 
mhapi__query_result__unpack(NULL, buf.pos, (const uint8_t *) buf.buf);
+   bs2_resetbuf(hdl->mid->from);
+   mnstr_readChr(hdl->mid->from, );
+   buf =  bs2_buffer(hdl->mid->from);
+   Mhapi__QueryResult *res = 
mhapi__query_result__unpack(NULL, buf.len + buf.pos, (const uint8_t *) buf.buf);
+   assert(res);
assert(res->row_count <= result->row_count);
assert(res->n_columns == (size_t) 
result->fieldcnt);
 
@@ -5600,7 +5605,7 @@ mapi_fetch_row(MapiHdl hdl)
result->tuple_count += res->row_count;
result->rows_read++;
if(hdl->mid->protobuf_res) {
-   free(hdl->mid->protobuf_res);
+   
mhapi__query_result__free_unpacked(NULL, hdl->mid->protobuf_res);
}
hdl->mid->protobuf_res = (void*) res;
return result->fieldcnt;
diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag
--- a/common/stream/Makefile.ag
+++ b/common/stream/Makefile.ag
@@ -31,9 +31,9 @@ lib_stream  =  {
   $(LTLIBICONV)
 }
 
-lib_stream_protobuf = {
+lib_protobuf = {
COND = HAVE_LIBPROTOBUF
-   SOURCES = mhapi.pb-c.h mhapi.pb-c.c
+   SOURCES = mhapi.c mhapi.h
VERSION = 42
LIBS =  $(protobuf_LIBS)
 }
diff --git a/common/stream/mhapi.c b/common/stream/mhapi.c
new file mode 100644
--- /dev/null
+++ b/common/stream/mhapi.c
@@ -0,0 +1,283 @@
+/* Generated by the protocol buffer compiler.  DO NOT EDIT! */
+/* Generated from: common/stream/mhapi.proto */
+
+/* Do not generate deprecated warnings for self */
+#ifndef PROTOBUF_C__NO_DEPRECATED
+#define PROTOBUF_C__NO_DEPRECATED
+#endif
+
+#include "common/stream/mhapi.h"
+void   mhapi__query_result__column__init
+ (Mhapi__QueryResult__Column *message)
+{
+  static Mhapi__QueryResult__Column init_value = 
MHAPI__QUERY_RESULT__COLUMN__INIT;
+  *message = init_value;
+}
+void   mhapi__query_result__column_unpacked__init
+ (Mhapi__QueryResult__ColumnUnpacked *message)
+{
+  static Mhapi__QueryResult__ColumnUnpacked init_value = 
MHAPI__QUERY_RESULT__COLUMN_UNPACKED__INIT;
+  *message = init_value;
+}
+void   mhapi__query_result__init
+ (Mhapi__QueryResult *message)
+{
+  static Mhapi__QueryResult init_value = MHAPI__QUERY_RESULT__INIT;
+  *message = init_value;
+}
+size_t mhapi__query_result__get_packed_size
+ (const Mhapi__QueryResult *message)
+{
+  assert(message->base.descriptor == __query_result__descriptor);
+  return protobuf_c_message_get_packed_size ((const 
ProtobufCMessage*)(message));
+}
+size_t mhapi__query_result__pack
+

MonetDB: protocol - protobuf result sets first part

2016-08-15 Thread Hannes Muehleisen

Changeset: 5210c24ef644 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5210c24ef644
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/Makefile.ag
clients/mapilib/mapi.c
common/stream/mhapi.proto
common/stream/stream.c
common/stream/stream.h
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

protobuf result sets first part


diffs (truncated from 362 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -2957,7 +2957,7 @@ usage(const char *prog, int xit)
fprintf(stderr, " -C version  | --compression=type specify compression 
method {snappy,lz4}\n");
fprintf(stderr, " -P version  | --protocol=version specify protocol 
version {prot9,prot10,prot10compressed}\n");
fprintf(stderr, " -B size | --blocksize=size   specify protocol 
block size (>= %d)\n", BLOCK);
-   fprintf(stderr, " -c colcomp  | --colcomp=type specify column 
compression type {none,pfor}");
+   fprintf(stderr, " -c colcomp  | --colcomp=type specify column 
compression type {none,pfor,protobuf}");
 
fprintf(stderr, " -H  | --history  load/save cmdline 
history (default off)\n");
fprintf(stderr, " -i  | --interactive[=tm] interpret `\\' 
commands on stdin, use time formatting {ms,s,m}\n");
diff --git a/clients/mapilib/Makefile.ag b/clients/mapilib/Makefile.ag
--- a/clients/mapilib/Makefile.ag
+++ b/clients/mapilib/Makefile.ag
@@ -13,8 +13,9 @@ lib_mapi = {
VERSION = $(MAPI_VERSION)
SOURCES = mapi.c mapi.rc
LIBS = $(SOCKET_LIBS) ../../common/stream/libstream \
+   ../../common/stream/libstream_protobuf \
../../common/options/libmoptions \
-   ../../common/utils/libmcrypt $(openssl_LIBS) $(pfor_LIBS)
+   ../../common/utils/libmcrypt $(openssl_LIBS) $(pfor_LIBS) 
$(protobuf_LIBS)
 }
 
 headers_mapi = {
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -913,6 +913,7 @@ struct MapiStruct {
compression_method comp;
column_compression colcomp;
size_t blocksize;
+   void* protobuf_res;
 
int trace;  /* Trace Mapi interaction */
int auto_commit;
@@ -952,6 +953,8 @@ struct MapiResultSet {
struct MapiRowBuf cache;
int commentonly;/* only comments seen so far */
mapi_int64 rows_read;
+   mapi_int64 cur_row;
+
 };
 
 struct MapiStatement {
@@ -1905,6 +1908,7 @@ mapi_new(void)
mid->protocol = protauto;
mid->colcomp = COLUMN_COMPRESSION_AUTO;
mid->blocksize = 128 * BLOCK; // 1 MB
+   mid->protobuf_res = NULL;
 
mid->cachelimit = 100;
mid->redircnt = 0;
@@ -2796,11 +2800,7 @@ mapi_reconnect(Mapi mid)
 mid->database == NULL ? "" : mid->database,
 prot_version == prot10 ? "PROT10" : 
"PROT10COMPR",
 comp == COMPRESSION_SNAPPY ? "SNAPPY" : 
(comp == COMPRESSION_LZ4 ? "LZ4" : ""),
-#ifdef HAVE_PFOR
-mid->colcomp == COLUMN_COMPRESSION_PFOR ? 
",HAVEPFOR" : "",
-#else
-"",
-#endif
+mid->colcomp == COLUMN_COMPRESSION_PFOR ? 
",HAVEPFOR" :(mid->colcomp == COLUMN_COMPRESSION_PROTOBUF ? ",PROTOBUF" : 
""),
mid->blocksize);
} else {
retval = snprintf(buf, BLOCK, 
"%s:%s:%s:%s:%s:\n",
@@ -4236,7 +4236,7 @@ read_into_cache(MapiHdl hdl, int lookahe
result->querytype = Q_TABLE;
result->tuple_count = 0;
result->rows_read = 0;
-
+   result->cur_row = 0;
 
for (i = 0; i < nr_cols; i++) {
lng col_info_length;
@@ -5533,6 +5533,10 @@ mapi_split_line(MapiHdl hdl)
return n;
 }
 
+#ifdef HAVE_LIBPROTOBUF
+#include 
+#endif
+
 int
 mapi_fetch_row(MapiHdl hdl)
 {
@@ -5558,6 +5562,8 @@ mapi_fetch_row(MapiHdl hdl)
if (result->rows_read >= result->tuple_count) {
// if our cache is empty, we read data from the socket
lng nrows = 0;
+   result->cur_row = 1;
+
// first we write a prompt to the server indicating 
that we want another block of the result set
 
 #ifdef CONTINUATION_MESSAGE
@@ -5569,8 +5575,39 @@ mapi_fetch_row(MapiHdl hdl)
}
 #endif
 
+
+   if (hdl->mid->colcomp == COLUMN_COMPRESSION_PROTOBUF) {
+

MonetDB: protocol - bypass seek_row

2016-08-12 Thread Hannes Muehleisen

Changeset: f48e4231e0e4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f48e4231e0e4
Modified Files:
clients/mapilib/mapi.c
Branch: protocol
Log Message:

bypass seek_row


diffs (22 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2637,7 +2637,6 @@ mapi_reconnect(Mapi mid)
*hash = '\0';
rest = hash + 1;
}
-
 #ifdef HAVE_LIBSNAPPY
if (strstr(hashes, "PROT10COMPR")) {
// both server and client support compressed protocol 
10; use compressed version 
@@ -4876,7 +4875,9 @@ MapiMsg
 mapi_seek_row(MapiHdl hdl, mapi_int64 rownr, int whence)
 {
struct MapiResultSet *result;
-
+   if (hdl->mid->protocol != prot9) {
+   return 0;
+   }
mapi_hdl_check(hdl, "mapi_seek_row");
result = hdl->result;
switch (whence) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - mc

2016-08-10 Thread Hannes Muehleisen

Changeset: ec350d106e3a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec350d106e3a
Modified Files:
common/stream/stream.c
Branch: protocol
Log Message:

mc


diffs (16 lines):

diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4321,10 +4321,11 @@ bs2_read(stream *ss, void *buf, size_t e
size_t uncompressed_length = s->bufsiz;
size_t m = 0;
char *buf = s->buf;
+   snappy_status ret;
+
if (s->comp != COMPRESSION_NONE) {
buf = s->compbuf;
}
-   snappy_status ret;
 
while (m < s->itotal) {
ssize_t bytes_read = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - gcc fix

2016-08-10 Thread Hannes Muehleisen

Changeset: 82612daf988a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=82612daf988a
Modified Files:
common/stream/stream.c
Branch: protocol
Log Message:

gcc fix


diffs (16 lines):

diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4238,10 +4238,11 @@ bs2_read(stream *ss, void *buf, size_t e
size_t uncompressed_length = s->bufsiz;
size_t m = 0;
char *buf = s->buf;
+   snappy_status ret;
+
if (s->comp != COMPRESSION_NONE) {
buf = s->compbuf;
}
-   snappy_status ret;
 
while (m < s->itotal) {
ssize_t bytes_read = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - merge

2016-08-10 Thread Hannes Muehleisen

Changeset: 7ad69cae4a83 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7ad69cae4a83
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

merge


diffs (284 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -823,7 +823,11 @@ CSVrenderer(MapiHdl hdl)
 
for (i = 0; i < fields; i++) {
s = mapi_fetch_field(hdl, i);
-   buffer_ptr = stpcpy(buffer_ptr, s);
+   if (s) {
+   buffer_ptr = stpcpy(buffer_ptr, s);
+   } else {
+   buffer_ptr = stpcpy(buffer_ptr, 
default_nullstring);
+   }
 
if (i != fields - 1) {
*buffer_ptr++ = *sep;
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -829,6 +829,7 @@ struct MapiColumn {
int columnlength;
int digits;
int scale;
+   void *null_value;
char* buffer_ptr;
char write_buf[50];
mapi_converter converter;
@@ -3993,6 +3994,8 @@ parse_header_line(MapiHdl hdl, char *lin
 */
 
 static char* mapi_convert_varchar(struct MapiColumn *col) {
+   if (strcmp(col->buffer_ptr, (char*)col->null_value) == 0) 
+   return NULL;
return col->buffer_ptr;
 }
 
@@ -4018,33 +4021,38 @@ static char* itoa(int i, char b[]){
 }
 
 static char* mapi_convert_int(struct MapiColumn *col) {
-   //sprintf(col->write_buf, "%d", *((int*) col->buffer_ptr));
+   if (*((int*) col->buffer_ptr) == *((int*)col->null_value)) return NULL;
itoa(*((int*) col->buffer_ptr), col->write_buf);
return (char*) col->write_buf;
 }
 
 static char* mapi_convert_lng(struct MapiColumn *col) {
+   if (*((lng*) col->buffer_ptr) == *((lng*)col->null_value)) return NULL;
sprintf(col->write_buf, "%lld", *((lng*) col->buffer_ptr));
return (char*) col->write_buf;
 }
 
 static char* mapi_convert_smallint(struct MapiColumn *col) {
+   if (*((short*) col->buffer_ptr) == *((short*)col->null_value)) return 
NULL;
sprintf(col->write_buf, "%hd", *((short*) col->buffer_ptr));
return (char*) col->write_buf;
 }
 
 static char* mapi_convert_tinyint(struct MapiColumn *col) {
+   if (*((signed char*) col->buffer_ptr) == *((signed 
char*)col->null_value)) return NULL;
sprintf(col->write_buf, "%hhd", *((signed char*) col->buffer_ptr));
return (char*) col->write_buf;
 }
 
 static char* mapi_convert_double(struct MapiColumn *col) {
+   if (*((double*) col->buffer_ptr) == *((double*)col->null_value)) return 
NULL;
//sprintf(col->write_buf, "%g", *((double*) col->buffer_ptr));
gcvt(*((double*) col->buffer_ptr), 2, col->write_buf);
return (char*) col->write_buf;
 }
 
 static char* mapi_convert_boolean(struct MapiColumn *col) {
+   if (*((signed char*) col->buffer_ptr) == *((signed 
char*)col->null_value)) return NULL;
if (*((signed char*) col->buffer_ptr) == 1) {
return "true";
}
@@ -4088,12 +4096,11 @@ static int CUMLEAPDAYS[13] = {
 static char*
 mapi_convert_date(struct MapiColumn *col){
int day, month, year;
-
date n = *((date*) col->buffer_ptr);
 
-   if (n == date_nil) {
-   return "NULL";
-   }
+   if (n == *((date*)col->null_value)) 
+   return NULL;
+
year = n / 365;
day = (n - year * 365) - leapyears(year >= 0 ? year - 1 : year);
if (n < 0) {
@@ -4197,16 +4204,13 @@ read_into_cache(MapiHdl hdl, int lookahe
result->querytype = Q_TABLE;
result->tuple_count = 0;
result->rows_read = 0;
-// result->errorstr = NULL;
-// result->cache.line = NULL;
-// result->next = NULL;
-// result->hdl = hdl;
 
 
for (i = 0; i < nr_cols; i++) {
lng col_info_length;
char *table_name, *col_name, *type_sql_name;
int typelen;
+   int null_len;
 
if (!mnstr_readLng(mid->from, 
_info_length)) {
return mid->error;
@@ -4225,6 +4229,21 @@ read_into_cache(MapiHdl hdl, int lookahe
!mnstr_readInt(mid->from, 
)) {
return mid->error;
}
+
+   if (!mnstr_readInt(mid->from, _len)) {
+

MonetDB: protocol - mc

2016-08-10 Thread Hannes Muehleisen

Changeset: 2b09aa6b9ecd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=2b09aa6b9ecd
Modified Files:
clients/mapilib/mapi.c
Branch: protocol
Log Message:

mc


diffs (11 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -3996,6 +3996,7 @@ static char* mapi_convert_varchar(struct
return col->buffer_ptr;
 }
 
+// classic stackoverflow programming
 static char* itoa(int i, char b[]){
 char const digit[] = "0123456789";
 char* p = b;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - performance tweaks for string conversions

2016-08-10 Thread Hannes Muehleisen

Changeset: d48024361f64 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d48024361f64
Modified Files:
clients/mapilib/mapi.c
Branch: protocol
Log Message:

performance tweaks for string conversions


diffs (67 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -3996,8 +3996,29 @@ static char* mapi_convert_varchar(struct
return col->buffer_ptr;
 }
 
+static char* itoa(int i, char b[]){
+char const digit[] = "0123456789";
+char* p = b;
+if(i<0){
+*p++ = '-';
+i *= -1;
+}
+int shifter = i;
+do{ //Move to where representation ends
+++p;
+shifter = shifter/10;
+}while(shifter);
+*p = '\0';
+do{ //Move back, inserting digits as u go
+*--p = digit[i%10];
+i = i/10;
+}while(i);
+return b;
+}
+
 static char* mapi_convert_int(struct MapiColumn *col) {
-   sprintf(col->write_buf, "%d", *((int*) col->buffer_ptr));
+   //sprintf(col->write_buf, "%d", *((int*) col->buffer_ptr));
+   itoa(*((int*) col->buffer_ptr), col->write_buf);
return (char*) col->write_buf;
 }
 
@@ -4017,7 +4038,8 @@ static char* mapi_convert_tinyint(struct
 }
 
 static char* mapi_convert_double(struct MapiColumn *col) {
-   sprintf(col->write_buf, "%g", *((double*) col->buffer_ptr));
+   //sprintf(col->write_buf, "%g", *((double*) col->buffer_ptr));
+   gcvt(*((double*) col->buffer_ptr), 2, col->write_buf);
return (char*) col->write_buf;
 }
 
@@ -4101,8 +4123,20 @@ mapi_convert_date(struct MapiColumn *col
}
day -= CUMDAYS[month - 1];
}
-
-   sprintf(col->write_buf, "%04d-%02d-%02d", year, month, day);
+   // -MM-DD
+   col->write_buf[0] = (year/1000) + 48;
+   col->write_buf[1] = ((year/100) % 10) + 48;
+   col->write_buf[2] = ((year/10) % 10) + 48;
+   col->write_buf[3] = (year % 10) + 48;
+   col->write_buf[4] = '-';
+   col->write_buf[5] = (month/10) + 48;
+   col->write_buf[6] = (month % 10) + 48;
+   col->write_buf[7] = '-';
+   col->write_buf[8] = (day/10) + 48;
+   col->write_buf[9] = (day % 10) + 48;
+   col->write_buf[10] = 0;
+
+   //sprintf(col->write_buf, "%04d-%02d-%02d", year, month, day);
 
return col->write_buf;
 }
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - some performance tweaks

2016-08-09 Thread Hannes Muehleisen

Changeset: 8b371f2e57ad for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8b371f2e57ad
Modified Files:
clients/mapiclient/mclient.c
Branch: protocol
Log Message:

some performance tweaks


diffs (49 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -806,7 +806,8 @@ CSVrenderer(MapiHdl hdl)
char *s;
char *sep = separator;
int i;
-
+   char buffer[10];
+   char *buffer_ptr;
if (csvheader) {
fields = mapi_get_field_count(hdl);
for (i = 0; i < fields; i++) {
@@ -818,8 +819,18 @@ CSVrenderer(MapiHdl hdl)
mnstr_printf(toConsole, "\n");
}
while (!mnstr_errnr(toConsole) && (fields = fetch_row(hdl)) != 0) {
+   buffer_ptr = buffer;
+
for (i = 0; i < fields; i++) {
s = mapi_fetch_field(hdl, i);
+   buffer_ptr = stpcpy(buffer_ptr, s);
+
+   if (i != 0) {
+   *buffer_ptr++ = *sep;
+   }
+
+   /*
+
if (s == NULL)
s = nullstring == default_nullstring ? "" : 
nullstring;
if (strchr(s, *sep) != NULL ||
@@ -853,10 +864,13 @@ CSVrenderer(MapiHdl hdl)
mnstr_write(toConsole, "\"", 1, 1);
} else
mnstr_printf(toConsole, "%s%s",
-i == 0 ? "" : sep, s);
+i == 0 ? "" : sep, s);*/
}
-   mnstr_printf(toConsole, "\n");
-   mnstr_flush(toConsole);
+   //mnstr_printf(toConsole, "\n");
+
+   *buffer_ptr++ = 0;
+   puts(buffer);
+
}
 }
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - now shipping all of lineitem

2016-08-09 Thread Hannes Muehleisen

Changeset: ed9c9e15c398 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ed9c9e15c398
Modified Files:
clients/mapilib/mapi.c
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

now shipping all of lineitem


diffs (228 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -4001,6 +4001,11 @@ static char* mapi_convert_int(struct Map
return (char*) col->write_buf;
 }
 
+static char* mapi_convert_lng(struct MapiColumn *col) {
+   sprintf(col->write_buf, "%lld", *((lng*) col->buffer_ptr));
+   return (char*) col->write_buf;
+}
+
 static char* mapi_convert_smallint(struct MapiColumn *col) {
sprintf(col->write_buf, "%hd", *((short*) col->buffer_ptr));
return (char*) col->write_buf;
@@ -4011,6 +4016,11 @@ static char* mapi_convert_tinyint(struct
return (char*) col->write_buf;
 }
 
+static char* mapi_convert_double(struct MapiColumn *col) {
+   sprintf(col->write_buf, "%g", *((double*) col->buffer_ptr));
+   return (char*) col->write_buf;
+}
+
 static char* mapi_convert_boolean(struct MapiColumn *col) {
if (*((signed char*) col->buffer_ptr) == 1) {
return "true";
@@ -4020,6 +4030,86 @@ static char* mapi_convert_boolean(struct
}
 }
 
+
+/* apologies for now */
+typedef int date;
+int date_nil = -1;
+
+static int
+leapyears(int year)
+{
+   /* count the 4-fold years that passed since jan-1-0 */
+   int y4 = year / 4;
+
+   /* count the 100-fold years */
+   int y100 = year / 100;
+
+   /* count the 400-fold years */
+   int y400 = year / 400;
+
+   return y4 + y400 - y100 + (year >= 0);  /* may be negative */
+}
+
+#define leapyear(y)((y) % 4 == 0 && ((y) % 100 != 0 || (y) % 400 
== 0))
+
+#define YEARDAYS(y)(leapyear(y) ? 366 : 365)
+
+static int CUMDAYS[13] = {
+   0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365
+};
+static int CUMLEAPDAYS[13] = {
+   0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366
+};
+
+
+static char*
+mapi_convert_date(struct MapiColumn *col){
+   int day, month, year;
+
+   date n = *((date*) col->buffer_ptr);
+
+   if (n == date_nil) {
+   return "NULL";
+   }
+   year = n / 365;
+   day = (n - year * 365) - leapyears(year >= 0 ? year - 1 : year);
+   if (n < 0) {
+   year--;
+   while (day >= 0) {
+   year++;
+   day -= YEARDAYS(year);
+   }
+   day = YEARDAYS(year) + day;
+   } else {
+   while (day < 0) {
+   year--;
+   day += YEARDAYS(year);
+   }
+   }
+
+   day++;
+   if (leapyear(year)) {
+   for (month = day / 31 == 0 ? 1 : day / 31; month <= 12; month++)
+   if (day > CUMLEAPDAYS[month - 1] && day <= 
CUMLEAPDAYS[month]) {
+   break;
+   }
+   day -= CUMLEAPDAYS[month - 1];
+   } else {
+   for (month = day / 31 == 0 ? 1 : day / 31; month <= 12; month++)
+   if (day > CUMDAYS[month - 1] && day <= CUMDAYS[month]) {
+   break;
+   }
+   day -= CUMDAYS[month - 1];
+   }
+
+   sprintf(col->write_buf, "%04d-%02d-%02d", year, month, day);
+
+   return col->write_buf;
+}
+
+
+/* end of apologies */
+
 static char* mapi_convert_unknown(struct MapiColumn *col) {
(void) col;
return "";
@@ -4117,6 +4207,14 @@ read_into_cache(MapiHdl hdl, int lookahe
result->fields[i].converter = 
(mapi_converter) mapi_convert_tinyint;
} else if (strcasecmp(type_sql_name, "boolean") 
== 0) {
result->fields[i].converter = 
(mapi_converter) mapi_convert_boolean;
+   } else if (strcasecmp(type_sql_name, "decimal") 
== 0) {
+   result->fields[i].converter = 
(mapi_converter) mapi_convert_double;
+   } else if (strcasecmp(type_sql_name, "double") 
== 0) {
+   result->fields[i].converter = 
(mapi_converter) mapi_convert_double;
+   } else if (strcasecmp(type_sql_name, "date") == 
0) {
+   result->fields[i].converter = 
(mapi_converter) mapi_convert_date;
+   } else if (strcasecmp(type_sql_name, "bigint") 
== 0) {
+   result->fields[i].converter = 
(mapi_converter) mapi_convert_lng;
} else {
result->fields[i].converter = 
(mapi_converter) mapi_convert_unknown;

MonetDB: protocol - result sets actually working

2016-08-09 Thread Hannes Muehleisen

Changeset: 31989c598f67 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=31989c598f67
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

result sets actually working


diffs (truncated from 395 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -856,6 +856,7 @@ CSVrenderer(MapiHdl hdl)
 i == 0 ? "" : sep, s);
}
mnstr_printf(toConsole, "\n");
+   mnstr_flush(toConsole);
}
 }
 
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -818,6 +818,9 @@
 
 #define MAPIBLKSIZE256 /* minimum buffer shipped */
 
+typedef char* (*mapi_converter)(void*);
+
+
 /* information about the columns in a result set */
 struct MapiColumn {
char *tablename;
@@ -826,7 +829,9 @@ struct MapiColumn {
int columnlength;
int digits;
int scale;
-   void* dataprot10;
+   char* buffer_ptr;
+   char write_buf[50];
+   mapi_converter converter;
 };
 
 /* information about bound columns */
@@ -2662,6 +2667,7 @@ mapi_reconnect(Mapi mid)
return mid->error;
}
}
+   mid->protocol = prot_version;
 
/* in rest now should be the byte order of the server */
byteo = rest;
@@ -2803,8 +2809,7 @@ mapi_reconnect(Mapi mid)
check_stream(mid, mid->to, "Could not send initial byte sequence", 
"mapi_reconnect", mid->error);
 
if (prot_version == prot10 || prot_version == prot10compressed) {
-
-   printf("Using protocol version %s.\n", prot_version == prot10  
? "PROT10" : "PROT10COMPR");
+   //printf("Using protocol version %s.\n", prot_version == prot10 
 ? "PROT10" : "PROT10COMPR");
assert(isa_block_stream(mid->to));
assert(isa_block_stream(mid->from));
 
@@ -3986,6 +3991,41 @@ parse_header_line(MapiHdl hdl, char *lin
lines may cause a new result set to be created in which case all
subsequent lines are added to that result set.
 */
+
+static char* mapi_convert_varchar(struct MapiColumn *col) {
+   return col->buffer_ptr;
+}
+
+static char* mapi_convert_int(struct MapiColumn *col) {
+   sprintf(col->write_buf, "%d", *((int*) col->buffer_ptr));
+   return (char*) col->write_buf;
+}
+
+static char* mapi_convert_smallint(struct MapiColumn *col) {
+   sprintf(col->write_buf, "%hd", *((short*) col->buffer_ptr));
+   return (char*) col->write_buf;
+}
+
+static char* mapi_convert_tinyint(struct MapiColumn *col) {
+   sprintf(col->write_buf, "%hhd", *((signed char*) col->buffer_ptr));
+   return (char*) col->write_buf;
+}
+
+static char* mapi_convert_boolean(struct MapiColumn *col) {
+   if (*((signed char*) col->buffer_ptr) == 1) {
+   return "true";
+   }
+   else {
+   return "false";
+   }
+}
+
+static char* mapi_convert_unknown(struct MapiColumn *col) {
+   (void) col;
+   return "";
+}
+
+
 static MapiMsg
 read_into_cache(MapiHdl hdl, int lookahead)
 {
@@ -4012,7 +4052,8 @@ read_into_cache(MapiHdl hdl, int lookahe
lng nr_rows;
lng nr_cols;
lng i;
-   result = malloc(sizeof(struct MapiResultSet));
+   result = new_result(hdl);
+
if (!result) {
// TODO: actually set mid->error :)
return mid->error;
@@ -4022,13 +4063,20 @@ read_into_cache(MapiHdl hdl, int lookahe
!mnstr_readLng(mid->from, _cols)) {
return mid->error;
}
-   fprintf(stderr, "result_set_id=%d, nr_rows=%llu, 
nr_cols=%lld\n", result_set_id, nr_rows, nr_cols);
+   //  fprintf(stderr, "result_set_id=%d, nr_rows=%llu, 
nr_cols=%lld\n", result_set_id, nr_rows, nr_cols);
result->fieldcnt = nr_cols;
+   result->maxfields = nr_cols;
result->row_count = nr_rows;
result->fields = malloc(sizeof(struct MapiColumn) * 
result->fieldcnt);
result->tableid = result_set_id;
result->querytype = Q_TABLE;
result->tuple_count = 0;
+   result->rows_read = 0;
+// result->errorstr = NULL;
+// result->cache.line = NULL;
+// result->next = NULL;
+// result->hdl =

MonetDB: protocol - client

2016-08-09 Thread Hannes Muehleisen

Changeset: e79f82d345d7 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e79f82d345d7
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

client


diffs (185 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -330,13 +330,16 @@ fetch_line(MapiHdl hdl)
 static int
 fetch_row(MapiHdl hdl)
 {
-   char *reply;
-
-   do {
-   if ((reply = fetch_line(hdl)) == NULL)
-   return 0;
-   } while (*reply != '[' && *reply != '=');
-   return mapi_split_line(hdl);
+   // yeah right. circumvent your own api...
+// char *reply;
+//
+// do {
+// if ((reply = fetch_line(hdl)) == NULL)
+// return 0;
+// } while (*reply != '[' && *reply != '=');
+// return mapi_split_line(hdl);
+//
+   return mapi_fetch_row(hdl);
 }
 
 static void
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -826,6 +826,7 @@ struct MapiColumn {
int columnlength;
int digits;
int scale;
+   void* dataprot10;
 };
 
 /* information about bound columns */
@@ -938,6 +939,7 @@ struct MapiResultSet {
struct MapiColumn *fields;
struct MapiRowBuf cache;
int commentonly;/* only comments seen so far */
+   mapi_int64 rows_read;
 };
 
 struct MapiStatement {
@@ -4010,13 +4012,23 @@ read_into_cache(MapiHdl hdl, int lookahe
lng nr_rows;
lng nr_cols;
lng i;
+   result = malloc(sizeof(struct MapiResultSet));
+   if (!result) {
+   // TODO: actually set mid->error :)
+   return mid->error;
+   }
if (!mnstr_readInt(mid->from, _set_id) ||
!mnstr_readLng(mid->from, _rows) ||
!mnstr_readLng(mid->from, _cols)) {
return mid->error;
}
fprintf(stderr, "result_set_id=%d, nr_rows=%llu, 
nr_cols=%lld\n", result_set_id, nr_rows, nr_cols);
-
+   result->fieldcnt = nr_cols;
+   result->row_count = nr_rows;
+   result->fields = malloc(sizeof(struct MapiColumn) * 
result->fieldcnt);
+   result->tableid = result_set_id;
+   result->querytype = Q_TABLE;
+   result->tuple_count = 0;
 
for (i = 0; i < nr_cols; i++) {
lng col_info_length;
@@ -4040,18 +4052,19 @@ read_into_cache(MapiHdl hdl, int lookahe
!mnstr_readInt(mid->from, 
)) {
return mid->error;
}
-   fprintf(stderr, "%lld col_info_length=%lld, 
table_name=%s, col_name=%s, type_sql_name=%s, type_len=%d\n", i, 
col_info_length, table_name, col_name, type_sql_name, typelen);
+   fprintf(stderr, "%lld col_info_length=%lld, 
table_name=%s, col_name=%s, type_sql_name=%s, type_len=%d\n",
+   i, col_info_length, table_name, 
col_name, type_sql_name, typelen);
+   result->fields[i].columnname = col_name;
+   result->fields[i].tablename = table_name;
+   result->fields[i].columntype = type_sql_name;
}
+   hdl->result = result;
+   hdl->active = result;
 
{
-   lng nrows = 0;
char dummy;
// we flush on the other side so this read will 
always fail
mnstr_readChr(mid->from, );
-   if (!mnstr_readLng(mid->from, )) {
-   return mid->error;
-   }
-   fprintf(stderr, "nrows=%llu\n", nrows);
}
 
 
@@ -5276,6 +5289,27 @@ mapi_fetch_row(MapiHdl hdl)
int n;
struct MapiResultSet *result;
 
+   if (hdl->mid->protocol == prot10) {
+   result = hdl->result;
+   result->rows_read++;
+   // do we have any rows in our cache
+   if (result->rows_read > result->tuple_count && 
result->rows_read < result->row_count) {
+   // read block from socket
+   lng nrows = 0;
+   // we flush on the other

MonetDB: protocol - Backed out changeset 9002fbfe0dac

2016-08-09 Thread Hannes Muehleisen

Changeset: 9786385f19cc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9786385f19cc
Modified Files:
bootstrap
common/Makefile.ag
configure.ag
monetdb5/mal/Makefile.ag
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_parser.h
monetdb5/mal/mal_session.c
monetdb5/mal/mal_session.h
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/remote.c
monetdb5/optimizer/opt_pipes.c
sql/backends/monet5/sql_scenario.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_scan.c
sql/server/sql_scan.h
Branch: protocol
Log Message:

Backed out changeset 9002fbfe0dac


diffs (truncated from 1066 to 300 lines):

diff --git a/bootstrap b/bootstrap
--- a/bootstrap
+++ b/bootstrap
@@ -59,5 +59,3 @@ autoconf
 if [ -f buildtools/conf/install-sh -a ! -x buildtools/conf/install-sh ]; then
 chmod +x buildtools/conf/install-sh
 fi
-
-protoc-c common/protobuf/mhapi.proto --c_out common/protobuf
diff --git a/common/Makefile.ag b/common/Makefile.ag
--- a/common/Makefile.ag
+++ b/common/Makefile.ag
@@ -4,4 +4,4 @@
 #
 # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
 
-SUBDIRS = stream options utils protobuf
+SUBDIRS = stream options utils
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1641,51 +1641,6 @@ AM_CONDITIONAL([HAVE_LIBSNAPPY], [test x
 
 
 
-dnl  check for protobuf-c serialization library
-org_have_protobuf=auto
-have_protobuf=$org_have_protobuf
-protobuf_CFLAGS=""
-protobuf_LIBS=""
-AC_ARG_WITH([protobuf],
-   [AS_HELP_STRING([--with-protobuf=DIR],
-   [protobuf library is installed in DIR])],
-   [have_protobuf="$withval"])
-
-AS_CASE(["$have_protobuf"],
-   [yes|no|auto], [],
-   [
-   protobuf_CFLAGS="-I$withval/include"
-   protobuf_LIBS="-L$withval/lib"])
-
-AC_MSG_CHECKING([for protobuf])
-AS_VAR_IF([have_protobuf], [no], [], [
-   save_CPPFLAGS="$CPPFLAGS"
-   CPPFLAGS="$CPPFLAGS $protobuf_CFLAGS"
-   save_LIBS="$LIBS"
-   LIBS="$LIBS $protobuf_LIBS -lprotobuf-c"
-   AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include 
-@%:@include ]], 
[[(void)protobuf_c_version_number();]])],
-   protobuf_LIBS="$protobuf_LIBS -lprotobuf-c",
-   [ AS_VAR_IF([have_protobuf], [auto], [], [
-   AC_MSG_ERROR([protobuf library not found])])
- have_protobuf=no
- why_have_protobuf="(protobuf library not found)" ])
-   LIBS="$save_LIBS"
-   CPPFLAGS="$save_CPPFLAGS"])
-
-AS_VAR_IF([have_protobuf], [no], [], [
-   AC_DEFINE([HAVE_LIBPROTOBUF], 1, [Define if you have the protobuf 
library])
-   AC_MSG_RESULT([yes: $protobuf_LIBS])], [
-   protobuf_CFLAGS=""
-   protobuf_LIBS=""
-   AC_MSG_RESULT([no])])
-
-AC_SUBST([protobuf_CFLAGS])
-AC_SUBST([protobuf_LIBS])
-AM_CONDITIONAL([HAVE_LIBPROTOBUF], [test x$have_protobuf != xno])
-
-
-
 DL_LIBS=""
 AC_CHECK_LIB([dl], [dlopen], [DL_LIBS="-ldl" ] )
 AC_SUBST([DL_LIBS])
@@ -3329,7 +3284,6 @@ for comp in \
'pcre ' \
'perl ' \
'proj ' \
-   'protobuf ' \
'pthread  ' \
'python2  ' \
'python3  ' \
diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag
--- a/monetdb5/mal/Makefile.ag
+++ b/monetdb5/mal/Makefile.ag
@@ -6,7 +6,6 @@
 
 INCLUDES = ../../common/options \
   ../../common/stream \
-  ../../common/protobuf \
   ../../common/utils \
   ../../clients/mapilib \
   ../../gdk \
@@ -38,6 +37,7 @@ lib_mal = {
mal_namespace.c mal_namespace.h \
mal_parser.c mal_parser.h \
mal_profiler.c mal_profiler.h \
+   mal_readline.c mal_readline.h \
mal_resolve.c mal_resolve.h \
mal_sabaoth.c mal_sabaoth.h \
mal_scenario.c mal_scenario.h \
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -42,6 +42,7 @@
 /* (author) M.L. Kersten */
 #include "monetdb_config.h"
 #include "mal_client.h"
+#include "mal_readline.h"
 #include "mal_import.h"
 #include "mal_parser.h"
 #include "mal_namespace.h"
@@ -93,7 +94,7 @@ MCinit(void)
 }
 
 int
-MCpushClientInput(Client c, stream *new_input, int listing, char *prompt)
+MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt)
 {
ClientInput *x = (ClientInput *) GDKmalloc(sizeof(ClientInput));
if (x == 0)
@@ -118,7 +119,7 @@ MCpopClientInput(Client c)

MonetDB: protocol - Backed out changeset d5d332081c12

2016-08-09 Thread Hannes Muehleisen

Changeset: 204f8ef1cfb4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=204f8ef1cfb4
Removed Files:
common/protobuf/Makefile.ag
common/protobuf/messages.c
common/protobuf/messages.h
common/protobuf/mhapi.proto
Modified Files:
monetdb5/modules/mal/mal_mapi.c
Branch: protocol
Log Message:

Backed out changeset d5d332081c12


diffs (truncated from 313 to 300 lines):

diff --git a/common/protobuf/Makefile.ag b/common/protobuf/Makefile.ag
deleted file mode 100644
--- a/common/protobuf/Makefile.ag
+++ /dev/null
@@ -1,25 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0.  If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
-
-## Process this file with automake to produce Makefile.in
-
-MTSAFE
-
-INCLUDES = ../stream \
-   $(snappy_CFLAGS) \
-   $(protobuf_CFLAGS) 
-
-lib_protobuf  =  {
-   SOURCES = messages.c messages.h mhapi.pb-c.c mhapi.pb-c.h
-   LIBS = $(snappy_LIBS) \
-  $(protobuf_LIBS)
-}
-
-headers_common = {
-   DIR = includedir/monetdb
-   HEADERS = h
-   SOURCES = messages.h mhapi.pb-c.h
-}
diff --git a/common/protobuf/messages.c b/common/protobuf/messages.c
deleted file mode 100644
--- a/common/protobuf/messages.c
+++ /dev/null
@@ -1,175 +0,0 @@
-#include "messages.h"
-#ifdef HAVE_LIBSNAPPY
-#include  // C forever
-#endif
-
-#include /* va_alist.. */
-
-
-Mhapi__Message* message_read(stream *s, compression_method comp) {
-   lng len;
-   mnstr_readLng(s, );
-   return message_read_length(s, comp, len);
-}
-
-Mhapi__Message* message_read_length(stream *s, compression_method comp, lng 
len) {
-   Mhapi__Message* ret;
-   char* read_buf;
-
-   if (len < 1) {
-   return NULL;
-   }
-   read_buf = malloc(len);
-   if (!read_buf) {
-   return NULL;
-   }
-   if (mnstr_read(s, read_buf, len, 1) != len) {
-   return NULL;
-   }
-   switch(comp) {
-   case COMPRESSION_NONE:
-   break;
-   case COMPRESSION_SNAPPY:
-#ifdef HAVE_LIBSNAPPY
-   {
-   size_t uncompressed_length;
-   char *uncompressed_buf;
-   if (!snappy_uncompressed_length(read_buf, len, 
_length) == SNAPPY_OK) {
-   free(read_buf);
-   return NULL;
-   }
-   uncompressed_buf = malloc(uncompressed_length);
-   if (!uncompressed_buf) {
-   free(read_buf);
-   return NULL;
-   }
-   if (snappy_uncompress(read_buf, len, uncompressed_buf, 
_length) != SNAPPY_OK) {
-   free(read_buf);
-   free(uncompressed_buf);
-   return NULL;
-   }
-   free(read_buf);
-   read_buf = uncompressed_buf;
-   len = uncompressed_length;
-   }
-#else
-   return NULL;
-#endif
-   }
-
-   ret = (Mhapi__Message*) 
protobuf_c_message_unpack(__message__descriptor, NULL, len, (uint8_t*) 
read_buf);
-   free(read_buf);
-   return ret;
-}
-
-ssize_t message_write(stream *s, compression_method comp, Mhapi__Message *msg) 
{
-   lng len = protobuf_c_message_get_packed_size((ProtobufCMessage*) msg);
-   char* write_buf = malloc(len);
-   if (!write_buf) {
-   return -1;
-   }
-   if (protobuf_c_message_pack((ProtobufCMessage*) msg, (uint8_t*) 
write_buf) != (size_t) len) {
-   return -1;
-   }
-   switch(comp) {
-   case COMPRESSION_NONE:
-   break;
-   case COMPRESSION_SNAPPY:
-#ifdef HAVE_LIBSNAPPY
-   {
-   size_t compressed_length = snappy_max_compressed_length(len);
-   char *compressed_buf = malloc(compressed_length);
-   if (!compressed_buf) {
-   free(write_buf);
-   return NULL;
-   }
-   if (snappy_compress(write_buf, len, compressed_buf, 
_length) != SNAPPY_OK) {
-   free(write_buf);
-   free(compressed_buf);
-   return NULL;
-   }
-   free(write_buf);
-   write_buf = compressed_buf;
-   len = compressed_length;
-   }
-#else
-   return -1;
-#endif
-   }
-
-   if (!mnstr_writeLng(s, len) || mnstr_write(s, write_buf, len, 1) != 
len) {
-   return -1;
-   }
-   return len;
-}
-
-void message_send_error(stream *s, protocol_version proto, const char *format, 
...) {
-   char buf[BUFSIZ], *bf = buf;
-   int i = 0;
-   va_list ap;
-
-   if (s == NULL) {
-   return;
-   }
-   va_start(ap, format);

MonetDB: protocol - Backed out changeset a24577108238

2016-08-09 Thread Hannes Muehleisen

Changeset: 814c5a3fad3b for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=814c5a3fad3b
Modified Files:
common/protobuf/mhapi.proto
Branch: protocol
Log Message:

Backed out changeset a24577108238


diffs (10 lines):

diff --git a/common/protobuf/mhapi.proto b/common/protobuf/mhapi.proto
--- a/common/protobuf/mhapi.proto
+++ b/common/protobuf/mhapi.proto
@@ -68,4 +68,4 @@ message Message {
Warning warning = 6;
QueryResult queryresult = 7;
}
-}
+}
\ No newline at end of file
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - wh

2016-08-09 Thread Hannes Muehleisen

Changeset: a24577108238 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a24577108238
Modified Files:
common/protobuf/mhapi.proto
Branch: protocol
Log Message:

wh


diffs (10 lines):

diff --git a/common/protobuf/mhapi.proto b/common/protobuf/mhapi.proto
--- a/common/protobuf/mhapi.proto
+++ b/common/protobuf/mhapi.proto
@@ -68,4 +68,4 @@ message Message {
Warning warning = 6;
QueryResult queryresult = 7;
}
-}
\ No newline at end of file
+}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: protocol - proto stuff

2016-08-09 Thread Hannes Muehleisen

Changeset: d5d332081c12 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d5d332081c12
Added Files:
common/protobuf/Makefile.ag
common/protobuf/messages.c
common/protobuf/messages.h
common/protobuf/mhapi.proto
Modified Files:
monetdb5/modules/mal/mal_mapi.c
Branch: protocol
Log Message:

proto stuff


diffs (truncated from 313 to 300 lines):

diff --git a/common/protobuf/Makefile.ag b/common/protobuf/Makefile.ag
new file mode 100644
--- /dev/null
+++ b/common/protobuf/Makefile.ag
@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0.  If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
+
+## Process this file with automake to produce Makefile.in
+
+MTSAFE
+
+INCLUDES = ../stream \
+   $(snappy_CFLAGS) \
+   $(protobuf_CFLAGS) 
+
+lib_protobuf  =  {
+   SOURCES = messages.c messages.h mhapi.pb-c.c mhapi.pb-c.h
+   LIBS = $(snappy_LIBS) \
+  $(protobuf_LIBS)
+}
+
+headers_common = {
+   DIR = includedir/monetdb
+   HEADERS = h
+   SOURCES = messages.h mhapi.pb-c.h
+}
diff --git a/common/protobuf/messages.c b/common/protobuf/messages.c
new file mode 100644
--- /dev/null
+++ b/common/protobuf/messages.c
@@ -0,0 +1,175 @@
+#include "messages.h"
+#ifdef HAVE_LIBSNAPPY
+#include  // C forever
+#endif
+
+#include /* va_alist.. */
+
+
+Mhapi__Message* message_read(stream *s, compression_method comp) {
+   lng len;
+   mnstr_readLng(s, );
+   return message_read_length(s, comp, len);
+}
+
+Mhapi__Message* message_read_length(stream *s, compression_method comp, lng 
len) {
+   Mhapi__Message* ret;
+   char* read_buf;
+
+   if (len < 1) {
+   return NULL;
+   }
+   read_buf = malloc(len);
+   if (!read_buf) {
+   return NULL;
+   }
+   if (mnstr_read(s, read_buf, len, 1) != len) {
+   return NULL;
+   }
+   switch(comp) {
+   case COMPRESSION_NONE:
+   break;
+   case COMPRESSION_SNAPPY:
+#ifdef HAVE_LIBSNAPPY
+   {
+   size_t uncompressed_length;
+   char *uncompressed_buf;
+   if (!snappy_uncompressed_length(read_buf, len, 
_length) == SNAPPY_OK) {
+   free(read_buf);
+   return NULL;
+   }
+   uncompressed_buf = malloc(uncompressed_length);
+   if (!uncompressed_buf) {
+   free(read_buf);
+   return NULL;
+   }
+   if (snappy_uncompress(read_buf, len, uncompressed_buf, 
_length) != SNAPPY_OK) {
+   free(read_buf);
+   free(uncompressed_buf);
+   return NULL;
+   }
+   free(read_buf);
+   read_buf = uncompressed_buf;
+   len = uncompressed_length;
+   }
+#else
+   return NULL;
+#endif
+   }
+
+   ret = (Mhapi__Message*) 
protobuf_c_message_unpack(__message__descriptor, NULL, len, (uint8_t*) 
read_buf);
+   free(read_buf);
+   return ret;
+}
+
+ssize_t message_write(stream *s, compression_method comp, Mhapi__Message *msg) 
{
+   lng len = protobuf_c_message_get_packed_size((ProtobufCMessage*) msg);
+   char* write_buf = malloc(len);
+   if (!write_buf) {
+   return -1;
+   }
+   if (protobuf_c_message_pack((ProtobufCMessage*) msg, (uint8_t*) 
write_buf) != (size_t) len) {
+   return -1;
+   }
+   switch(comp) {
+   case COMPRESSION_NONE:
+   break;
+   case COMPRESSION_SNAPPY:
+#ifdef HAVE_LIBSNAPPY
+   {
+   size_t compressed_length = snappy_max_compressed_length(len);
+   char *compressed_buf = malloc(compressed_length);
+   if (!compressed_buf) {
+   free(write_buf);
+   return NULL;
+   }
+   if (snappy_compress(write_buf, len, compressed_buf, 
_length) != SNAPPY_OK) {
+   free(write_buf);
+   free(compressed_buf);
+   return NULL;
+   }
+   free(write_buf);
+   write_buf = compressed_buf;
+   len = compressed_length;
+   }
+#else
+   return -1;
+#endif
+   }
+
+   if (!mnstr_writeLng(s, len) || mnstr_write(s, write_buf, len, 1) != 
len) {
+   return -1;
+   }
+   return len;
+}
+
+void message_send_error(stream *s, protocol_version proto, const char *format, 
...) {
+   char buf[BUFSIZ], *bf = buf;
+   int i = 0;
+   va_list ap;
+
+   if (s == NULL) {
+   return;
+   }
+   va_start(ap, format);
+   i = vsnprintf(bf,

MonetDB: protocol - protobuf prototype and bstream zapping

2016-08-08 Thread Hannes Muehleisen

Changeset: 9002fbfe0dac for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9002fbfe0dac
Modified Files:
bootstrap
common/Makefile.ag
configure.ag
monetdb5/mal/Makefile.ag
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_import.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_parser.c
monetdb5/mal/mal_parser.h
monetdb5/mal/mal_session.c
monetdb5/mal/mal_session.h
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/remote.c
monetdb5/optimizer/opt_pipes.c
sql/backends/monet5/sql_scenario.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_scan.c
sql/server/sql_scan.h
Branch: protocol
Log Message:

protobuf prototype and bstream zapping


diffs (truncated from 1067 to 300 lines):

diff --git a/bootstrap b/bootstrap
--- a/bootstrap
+++ b/bootstrap
@@ -59,3 +59,5 @@ autoconf
 if [ -f buildtools/conf/install-sh -a ! -x buildtools/conf/install-sh ]; then
 chmod +x buildtools/conf/install-sh
 fi
+
+protoc-c common/protobuf/mhapi.proto --c_out common/protobuf
diff --git a/common/Makefile.ag b/common/Makefile.ag
--- a/common/Makefile.ag
+++ b/common/Makefile.ag
@@ -4,4 +4,4 @@
 #
 # Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
 
-SUBDIRS = stream options utils
+SUBDIRS = stream options utils protobuf
diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -1641,6 +1641,51 @@ AM_CONDITIONAL([HAVE_LIBSNAPPY], [test x
 
 
 
+dnl  check for protobuf-c serialization library
+org_have_protobuf=auto
+have_protobuf=$org_have_protobuf
+protobuf_CFLAGS=""
+protobuf_LIBS=""
+AC_ARG_WITH([protobuf],
+   [AS_HELP_STRING([--with-protobuf=DIR],
+   [protobuf library is installed in DIR])],
+   [have_protobuf="$withval"])
+
+AS_CASE(["$have_protobuf"],
+   [yes|no|auto], [],
+   [
+   protobuf_CFLAGS="-I$withval/include"
+   protobuf_LIBS="-L$withval/lib"])
+
+AC_MSG_CHECKING([for protobuf])
+AS_VAR_IF([have_protobuf], [no], [], [
+   save_CPPFLAGS="$CPPFLAGS"
+   CPPFLAGS="$CPPFLAGS $protobuf_CFLAGS"
+   save_LIBS="$LIBS"
+   LIBS="$LIBS $protobuf_LIBS -lprotobuf-c"
+   AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include 
+@%:@include ]], 
[[(void)protobuf_c_version_number();]])],
+   protobuf_LIBS="$protobuf_LIBS -lprotobuf-c",
+   [ AS_VAR_IF([have_protobuf], [auto], [], [
+   AC_MSG_ERROR([protobuf library not found])])
+ have_protobuf=no
+ why_have_protobuf="(protobuf library not found)" ])
+   LIBS="$save_LIBS"
+   CPPFLAGS="$save_CPPFLAGS"])
+
+AS_VAR_IF([have_protobuf], [no], [], [
+   AC_DEFINE([HAVE_LIBPROTOBUF], 1, [Define if you have the protobuf 
library])
+   AC_MSG_RESULT([yes: $protobuf_LIBS])], [
+   protobuf_CFLAGS=""
+   protobuf_LIBS=""
+   AC_MSG_RESULT([no])])
+
+AC_SUBST([protobuf_CFLAGS])
+AC_SUBST([protobuf_LIBS])
+AM_CONDITIONAL([HAVE_LIBPROTOBUF], [test x$have_protobuf != xno])
+
+
+
 DL_LIBS=""
 AC_CHECK_LIB([dl], [dlopen], [DL_LIBS="-ldl" ] )
 AC_SUBST([DL_LIBS])
@@ -3284,6 +3329,7 @@ for comp in \
'pcre ' \
'perl ' \
'proj ' \
+   'protobuf ' \
'pthread  ' \
'python2  ' \
'python3  ' \
diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag
--- a/monetdb5/mal/Makefile.ag
+++ b/monetdb5/mal/Makefile.ag
@@ -6,6 +6,7 @@
 
 INCLUDES = ../../common/options \
   ../../common/stream \
+  ../../common/protobuf \
   ../../common/utils \
   ../../clients/mapilib \
   ../../gdk \
@@ -37,7 +38,6 @@ lib_mal = {
mal_namespace.c mal_namespace.h \
mal_parser.c mal_parser.h \
mal_profiler.c mal_profiler.h \
-   mal_readline.c mal_readline.h \
mal_resolve.c mal_resolve.h \
mal_sabaoth.c mal_sabaoth.h \
mal_scenario.c mal_scenario.h \
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -42,7 +42,6 @@
 /* (author) M.L. Kersten */
 #include "monetdb_config.h"
 #include "mal_client.h"
-#include "mal_readline.h"
 #include "mal_import.h"
 #include "mal_parser.h"
 #include "mal_namespace.h"
@@ -94,7 +93,7 @@ MCinit(void)
 }
 
 int
-MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt)
+MCpushClientInput(Client c, stream *new_input, int listing, char *prompt)
 {
ClientInput *x = (ClientInput *) GDKmalloc(sizeof(ClientInput));
if (x == 0)
@@ -119,7 +118,7 @@ MCpopClientInput(Client c)

MonetDB: protocol - new result set first part and mapi read_line...

2016-08-05 Thread Hannes Muehleisen

Changeset: 8155970591cd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8155970591cd
Modified Files:
clients/mapilib/mapi.c
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/sql_result.c
sql/include/sql_query.h
Branch: protocol
Log Message:

new result set first part and mapi read_line cleanup


diffs (truncated from 430 to 300 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -876,9 +876,7 @@ struct MapiRowBuf {
 struct BlockCache {
char *buf;
int lim;
-   int nxt;
int end;
-   int eos;/* end of sequence */
 };
 
 
@@ -1894,18 +1892,12 @@ mapi_new(void)
mid->redircnt = 0;
mid->redirmax = 10;
mid->tracelog = NULL;
-   mid->blk.eos = 0;
-   mid->blk.buf = malloc(BLOCK + 1);
+   mid->blk.buf = malloc(BLOCK);
if (mid->blk.buf == NULL) {
mapi_destroy(mid);
return NULL;
}
-   mid->blk.buf[BLOCK] = 0;
-   mid->blk.buf[0] = 0;
-   mid->blk.nxt = 0;
-   mid->blk.end = 0;
mid->blk.lim = BLOCK;
-
mid->first = NULL;
 
return mid;
@@ -2558,7 +2550,7 @@ mapi_reconnect(Mapi mid)
/* consume server challenge */
len = mnstr_read_block(mid->from, buf, 1, BLOCK);
 
-   check_stream(mid, mid->from, "Connection terminated while starting", 
"mapi_reconnect", (mid->blk.eos = 1, mid->error));
+   check_stream(mid, mid->from, "Connection terminated while starting", 
"mapi_reconnect", mid->error);
 
assert(len < BLOCK);
buf[len] = 0;
@@ -3491,82 +3483,32 @@ mapi_param_store(MapiHdl hdl)
 static char *
 read_line(Mapi mid)
 {
-   char *reply;
-   char *nl;
-   char *s;/* from where to search for newline */
-
+   int ret = 0;
if (mid->active == NULL)
return 0;
-
-   /* check if we need to read more blocks to get a new line */
-   mid->blk.eos = 0;
-   s = mid->blk.buf + mid->blk.nxt;
-   while ((nl = strchr(s, '\n')) == NULL && !mid->blk.eos) {
-   ssize_t len;
-
-   if (mid->blk.lim - mid->blk.end < BLOCK) {
-   int len;
-
-   len = mid->blk.lim;
-   if (mid->blk.nxt <= BLOCK) {
-   /* extend space */
-   len += BLOCK;
+   mid->blk.end = 0;
+   do {
+   if ((mid->blk.end + 1) == mid->blk.lim) {
+   REALLOC(mid->blk.buf, mid->blk.lim + BLOCK);
+   if (!mid->blk.buf) {
+   return 0;
}
-   REALLOC(mid->blk.buf, len + 1);
-   if (mid->blk.nxt > 0) {
-   memmove(mid->blk.buf, mid->blk.buf + 
mid->blk.nxt, mid->blk.end - mid->blk.nxt + 1);
-   mid->blk.end -= mid->blk.nxt;
-   mid->blk.nxt = 0;
+   mid->blk.lim += BLOCK;
+   }
+   /* mid->from is **always** buffered, so no point in rolling an 
additional cache on top */
+   if ((ret = mnstr_readChr(mid->from, mid->blk.buf + 
mid->blk.end)) != 1) {
+   if (ret == 0) {
+   mid->blk.buf[0] = PROMPTBEG;
+   mid->blk.buf[1] = '\n';
+   mid->blk.buf[2] = 0;
+   return mid->blk.buf;
}
-   mid->blk.lim = len;
+   return 0;
}
-
-   s = mid->blk.buf + mid->blk.end;
-
-   /* fetch one more block */
-   if (mid->trace == MAPI_TRACE)
-   printf("fetch next block: start at:%d\n", mid->blk.end);
-   len = mnstr_read(mid->from, mid->blk.buf + mid->blk.end, 1, 
BLOCK);
-   check_stream(mid, mid->from, "Connection terminated during read 
line", "read_line", (mid->blk.eos = 1, (char *) 0));
-   if (mid->tracelog) {
-   mapi_log_header(mid, "R");
-   mnstr_write(mid->tracelog, mid->blk.buf + mid->blk.end, 
1, len);
-   mnstr_flush(mid->tracelog);
-   }
-   mid->blk.buf[mid->blk.end + len] = 0;
-   if (mid->trace == MAPI_TRACE) {
-   printf("got next block: length:" SSZFMT "\n", len);
-   printf("text:%s\n", mid->blk.buf + mid->blk.end);
-   }
-   if (len == 0) { /* add prompt */
-   if (mid->blk.end > mid->blk.nxt) {
-   /* add fake newline since newline was
-* missing from server */
-

MonetDB: protocol - compressed streams actually working for larg...

2016-08-05 Thread Hannes Muehleisen

Changeset: 7a7bd9024d74 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7a7bd9024d74
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/mapilib/mapi.h
common/stream/stream.c
common/stream/stream.h
monetdb5/mal/mal_client.c
monetdb5/mal/mal_client.h
monetdb5/mal/mal_function.c
monetdb5/mal/mal_session.c
monetdb5/mal/mal_session.h
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_result.c
sql/backends/monet5/sql_result.h
Branch: protocol
Log Message:

compressed streams actually working for large results


diffs (truncated from 479 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -3300,17 +3300,8 @@ main(int argc, char **argv)
}
 
if (protocol) {
-   if (strcasecmp(protocol, "prot9") == 0) {
-   mapi_set_protocol(mid, prot9);
-   }
-   else if (strcasecmp(protocol, "prot10") == 0) {
-   mapi_set_protocol(mid, prot10);
-   }
-   else if (strcasecmp(protocol, "prot10compressed") == 0) {
-   mapi_set_protocol(mid, prot10compressed);
-   }
-   else {
-   fprintf(stderr, "invalid protocol name '%s'\n", 
protocol);
+   if (mapi_set_protocol(mid, protocol) != 0) {
+   fprintf(stderr, "%s\n", mapi_error_str(mid));
}
}
 
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -5540,8 +5540,22 @@ mapi_get_active(Mapi mid)
return mid->active;
 }
 
-void mapi_set_protocol(Mapi mid, protocol_version prot) {
-   mid->protocol = prot;
+MapiMsg mapi_set_protocol(Mapi mid, const char* protocol) {
+   if (strcasecmp(protocol, "prot9") == 0) {
+   mid->protocol = prot9;
+   }
+   else if (strcasecmp(protocol, "prot10") == 0) {
+   mid->protocol = prot10;
+   }
+   else if (strcasecmp(protocol, "prot10compressed") == 0) {
+   mid->protocol = prot10compressed;
+   }
+   else {
+   mapi_setError(mid, "invalid protocol name", 
"mapi_set_protocol", MERROR);
+   return -1;
+   }
+
+   return 0;
 }
 
 void mapi_set_blocksize(Mapi mid, size_t blocksize) {
diff --git a/clients/mapilib/mapi.h b/clients/mapilib/mapi.h
--- a/clients/mapilib/mapi.h
+++ b/clients/mapilib/mapi.h
@@ -129,13 +129,6 @@ typedef struct {   /* used by MAPI_DATETI
unsigned int fraction;  /* in 1000 millionths of a second (10e-9) */
 } MapiDateTime;
 
-typedef enum {
-   protauto = 0,
-   prot9 = 1,
-   prot10 = 2,
-   prot10compressed = 3,
-} protocol_version;
-
 /* connection-oriented functions */
 mapi_export Mapi mapi_mapi(const char *host, int port, const char *username, 
const char *password, const char *lang, const char *dbname);
 mapi_export Mapi mapi_mapiuri(const char *url, const char *user, const char 
*pass, const char *lang);
@@ -240,7 +233,7 @@ mapi_export char *mapi_quote(const char 
 mapi_export char *mapi_unquote(char *msg);
 mapi_export MapiHdl mapi_get_active(Mapi mid);
 
-mapi_export void mapi_set_protocol(Mapi mid, protocol_version prot);
+mapi_export MapiMsg mapi_set_protocol(Mapi mid, const char* prot);
 mapi_export void mapi_set_blocksize(Mapi mid, size_t blocksize);
 
 #ifdef _MSC_VER
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -3977,6 +3977,7 @@ typedef struct bs2 {
size_t nr;  /* how far we got in buf */
size_t itotal;  /* amount available in current read block */
size_t bufsiz;
+   size_t readpos;
compression_method comp;
char *compbuf;
size_t compbufsiz;
@@ -4047,6 +4048,7 @@ bs2_write(stream *ss, const void *buf, s
s->nr += n;
todo -= n;
buf = ((const char *) buf + n);
+   /* block is full, write it to the stream */
if (s->nr == s->bufsiz) {
 
 #ifdef BSTREAM_DEBUG
@@ -4063,17 +4065,15 @@ bs2_write(stream *ss, const void *buf, s
}
 #endif
 
-
-   /* block is full, write it to the stream */
-
writelen = s->nr;
blksize = s->nr;
writebuf = s->buf;
 
if (s->comp == COMPRESSION_SNAPPY) {
+   snappy_status ret;
size_t compressed_length = s->compbufsiz;
-   if (snappy_compress(s->buf, s->nr, s->compbuf, 
_length) != SNAPPY_OK) {
-

MonetDB: protocol - merge with default

2016-08-05 Thread Hannes Muehleisen

Changeset: edcb9d8a2c8f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=edcb9d8a2c8f
Removed Files:
monetdb5/modules/mal/Tests/zorder.malC
monetdb5/modules/mal/Tests/zorder.stable.err
monetdb5/modules/mal/Tests/zorder.stable.out
monetdb5/modules/mal/zorder.c
monetdb5/modules/mal/zorder.h
monetdb5/modules/mal/zorder.mal
sql/scripts/24_zorder.sql
Modified Files:
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
clients/Tests/exports.stable.out
clients/odbc/driver/README
clients/odbc/driver/SQLBindParameter.c
clients/odbc/driver/SQLBulkOperations.c
clients/odbc/driver/SQLCopyDesc.c
clients/odbc/driver/SQLDescribeParam.c
clients/odbc/driver/SQLEndTran.c
clients/odbc/driver/SQLExtendedFetch.c
clients/odbc/driver/SQLFetchScroll.c
clients/odbc/driver/SQLGetEnvAttr.c
clients/odbc/driver/SQLNumParams.c
clients/odbc/driver/SQLSetDescRec.c
clients/odbc/driver/SQLSetEnvAttr.c
configure.ag
java/src/main/java/nl/cwi/monetdb/jdbc/MonetBlob.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetDriver.java.in
monetdb5/ChangeLog
monetdb5/mal/mal.h
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_instruction.h
monetdb5/mal/mal_listing.c
monetdb5/mal/mal_module.c
monetdb5/mal/mal_module.h
monetdb5/mal/mal_parser.c
monetdb5/modules/mal/Makefile.ag
monetdb5/modules/mal/Tests/All
monetdb5/modules/mal/mal_init.mal
monetdb5/optimizer/opt_profiler.c
rpm.mk.in
sql/ChangeLog
sql/backends/monet5/sql_scenario.c
sql/backends/monet5/sql_upgrades.c
sql/scripts/Makefile.ag
sql/test/BugTracker-2016/Tests/All
sql/test/BugTracker/Tests/jdbc_no_debug.SF-1739356.stable.out.32bit
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/check.stable.out
sql/test/emptydb-upgrade-chain/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/check.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/check.stable.out
sql/test/emptydb-upgrade/Tests/check.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/emptydb/Tests/check.stable.out
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.stable.out.int128
sql/test/leaks/Tests/check4.stable.out.int128
sql/test/leaks/Tests/check5.stable.out.int128
sql/test/pg_regress/Tests/strings.stable.err
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.32bit
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: protocol
Log Message:

merge with default


diffs (truncated from 5860 to 300 lines):

diff --git a/NT/installer32/MonetDB5-SQL-Installer.vdproj 
b/NT/installer32/MonetDB5-SQL-Installer.vdproj
--- a/NT/installer32/MonetDB5-SQL-Installer.vdproj
+++ b/NT/installer32/MonetDB5-SQL-Installer.vdproj
@@ -897,12 +897,6 @@
 }
 "Entry"
 {
-"MsmKey" = "8:_EC56B031A65A44BCA049E43F5A7641C4"
-"OwnerKey" = "8:_UNDEFINED"
-"MsmSig" = "8:_UNDEFINED"
-}
-"Entry"
-{
 "MsmKey" = "8:_F0AD495DD01A481B8FDB22499521AFB7"
 "OwnerKey" = "8:_UNDEFINED"
 "MsmSig" = "8:_UNDEFINED"
@@ -939,12 +933,6 @@
 }
 "Entry"
 {
-"MsmKey" = "8:_F64E69D474DA4F6A89A1B014F77E4669"
-"OwnerKey" = "8:_UNDEFINED"
-"MsmSig" = "8:_UNDEFINED"
-}
-"Entry"
-{
 "MsmKey" = "8:_F827A21B394243CA9A2DA27D42DA0C93"
 "OwnerKey" = "8:_UNDEFINED"
 "MsmSig" = "8:_UNDEFINED"
@@ -3964,26 +3952,6 @@

MonetDB: protocol - mclient support for new block stream, compre...

2016-08-05 Thread Hannes Muehleisen

Changeset: 721c8b3d945f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=721c8b3d945f
Modified Files:
clients/mapiclient/mclient.c
clients/mapilib/mapi.c
clients/mapilib/mapi.h
common/stream/stream.c
common/stream/stream.h
monetdb5/modules/mal/mal_mapi.c
Branch: protocol
Log Message:

mclient support for new block stream, compression working


diffs (truncated from 615 to 300 lines):

diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -2932,6 +2932,10 @@ usage(const char *prog, int xit)
fprintf(stderr, " -E charset  | --encoding=charset specify encoding 
(character set) of the terminal\n");
 #endif
fprintf(stderr, " -f kind | --format=kind  specify output 
format {csv,tab,raw,sql,xml}\n");
+
+   fprintf(stderr, " -P version  | --protocol=version specify protocol 
version {prot9,prot10,prot10compressed}\n");
+   fprintf(stderr, " -B size | --blocksize=size   specify protocol 
block size (>= %d)\n", BLOCK);
+
fprintf(stderr, " -H  | --history  load/save cmdline 
history (default off)\n");
fprintf(stderr, " -i  | --interactive[=tm] interpret `\\' 
commands on stdin, use time formatting {ms,s,m}\n");
fprintf(stderr, " -l language | --language=lang{sql,mal}\n");
@@ -2969,6 +2973,8 @@ main(int argc, char **argv)
char *command = NULL;
char *dbname = NULL;
char *output = NULL;/* output format as string */
+   char *protocol = NULL;
+   size_t blocksize = 0;
FILE *fp = NULL;
int trace = 0;
int dump = 0;
@@ -2992,6 +2998,9 @@ main(int argc, char **argv)
{"encoding", 1, 0, 'E'},
 #endif
{"format", 1, 0, 'f'},
+   {"protocol", 1, 0, 'P'},
+   {"blocksize", 1, 0, 'B'},
+
{"help", 0, 0, '?'},
{"history", 0, 0, 'H'},
{"host", 1, 0, 'h'},
@@ -3125,6 +3134,16 @@ main(int argc, char **argv)
free(output);
output = strdup(optarg);/* output format */
break;
+   case 'P':
+   assert(optarg);
+   if (protocol != NULL)
+   free(protocol);
+   protocol = strdup(optarg);
+   break;
+   case 'B':
+   assert(optarg);
+   blocksize = (size_t) atol(optarg);
+   break;
case 'i':
interactive = 1;
showtiming = 1;
@@ -3271,6 +3290,30 @@ main(int argc, char **argv)
if (passwd)
free(passwd);
passwd = NULL;
+
+   if (blocksize > 0) {
+   if (blocksize < BLOCK) {
+   fprintf(stderr, "invalid block size (needs to be bigger 
than %d)\n", BLOCK);
+   } else {
+   mapi_set_blocksize(mid, blocksize);
+   }
+   }
+
+   if (protocol) {
+   if (strcasecmp(protocol, "prot9") == 0) {
+   mapi_set_protocol(mid, prot9);
+   }
+   else if (strcasecmp(protocol, "prot10") == 0) {
+   mapi_set_protocol(mid, prot10);
+   }
+   else if (strcasecmp(protocol, "prot10compressed") == 0) {
+   mapi_set_protocol(mid, prot10compressed);
+   }
+   else {
+   fprintf(stderr, "invalid protocol name '%s'\n", 
protocol);
+   }
+   }
+
if (mid && mapi_error(mid) == MOK)
mapi_reconnect(mid);/* actually, initial connect */
 
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -881,6 +881,7 @@ struct BlockCache {
int eos;/* end of sequence */
 };
 
+
 /* A connection to a server is represented by a struct MapiStruct.  An
application can have any number of connections to any number of
servers.  Connections are completely independent of each other.
@@ -897,6 +898,8 @@ struct MapiStruct {
char *uri;
int languageId;
char *motd; /* welcome message from server */
+   protocol_version protocol;
+   size_t blocksize;
 
int trace;  /* Trace Mapi interaction */
int auto_commit;
@@ -1884,6 +1887,9 @@ mapi_new(void)
mid->username = NULL;
mid->password = NULL;
 
+   mid->protocol = protauto;
+   mid->blocksize = 128*BLOCK; // 1 MB
+
mid->cachelimit = 100;
mid->redircnt = 0;
mid->redirmax = 10;
@@ -2193,11 +2199,6 @@ mapi_destroy(Mapi mid)
return MOK;
 }
 
-typedef enum {
-   prot9 = 1,
-

MonetDB: protocol - mapi version 10, bigger and snappier blocks

2016-08-04 Thread Hannes Muehleisen

Changeset: a2e6bf7213e6 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a2e6bf7213e6
Modified Files:
clients/mapilib/mapi.c
common/stream/Makefile.ag
common/stream/stream.c
common/stream/stream.h
configure.ag
monetdb5/modules/mal/mal_mapi.c
Branch: protocol
Log Message:

mapi version 10, bigger and snappier blocks


diffs (truncated from 894 to 300 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2214,6 +2214,8 @@ mapi_reconnect(Mapi mid)
char *protover;
char *rest;
protocol_version prot_version = prot9;
+   // FIXME: make this configurable
+   size_t block_size = 1024000;
 
if (mid->connected)
close_connection(mid);
@@ -2732,7 +2734,7 @@ mapi_reconnect(Mapi mid)
if (prot_version == prot10 || prot_version == 
prot10compressed) {
// if we are using protocol 10, we have to send 
either PROT10/PROT10COMPRESSED to the server
// so the server knows which protocol to use
-   retval = snprintf(buf, BLOCK, 
"%s:%s:%s:%s:%s:%s:\n",
+   retval = snprintf(buf, BLOCK, 
"%s:%s:%s:%s:%s:%s:%zu:\n",
#ifdef WORDS_BIGENDIAN
 "BIG",
#else
@@ -2740,7 +2742,8 @@ mapi_reconnect(Mapi mid)
#endif
 mid->username, hash, mid->language,
 mid->database == NULL ? "" : mid->database,
-prot_version == prot10 ? "PROT10" : 
"PROT10COMPRESSED");
+prot_version == prot10 ? "PROT10" : 
"PROT10COMPRESSED",
+   block_size);
} else {
retval = snprintf(buf, BLOCK, 
"%s:%s:%s:%s:%s:\n",
#ifdef WORDS_BIGENDIAN
@@ -2781,15 +2784,13 @@ mapi_reconnect(Mapi mid)
check_stream(mid, mid->to, "Could not send initial byte sequence", 
"mapi_reconnect", mid->error);
 
if (prot_version == prot10 || prot_version == prot10compressed) {
-   bstream *bs_to = (bstream*) mid->to;
-   bstream *bs_from = (bstream*) mid->from;
 
printf("Using protocol version %s.\n", prot_version == prot10  
? "PROT10" : "PROT10COMPRESSED");
assert(isa_block_stream(mid->to));
assert(isa_block_stream(mid->from));
 
if (prot_version == prot10compressed) {
-#ifdef HAVE_LIBSNAPPY
+#ifdef HAVE_LIBSNAPPY2
mid->to = compressed_stream(bs_to->s, 
COMPRESSION_SNAPPY);
mid->from = compressed_stream(bs_from->s, 
COMPRESSION_SNAPPY);
 #else
@@ -2797,14 +2798,15 @@ mapi_reconnect(Mapi mid)
 #endif
} else {
// FIXME: figure out proper stream sizes
-   mid->to = byte_stream(bs_to->s, 1024000);
-   mid->from = byte_stream(bs_from->s, 1024000);
-
+   mid->to = block_stream2(bs_stream(mid->to), block_size);
+   mid->from = block_stream2(bs_stream(mid->from), 
block_size);
}
-   bs_to->s = NULL;
-   bs_from->s = NULL;
-   close_stream((stream*) bs_to);
-   close_stream((stream*) bs_from);
+
+   // FIXME: this leaks
+// bs_to->s = NULL;
+// bs_from->s = NULL;
+// close_stream((stream*) bs_to);
+// close_stream((stream*) bs_from);
 
}
 
diff --git a/common/stream/Makefile.ag b/common/stream/Makefile.ag
--- a/common/stream/Makefile.ag
+++ b/common/stream/Makefile.ag
@@ -10,6 +10,7 @@ MTSAFE
 
 INCLUDES = $(zlib_CFLAGS) \
   $(BZ_CFLAGS) \
+  $(snappy_CFLAGS) \
   $(liblzma_CFLAGS) \
   $(openssl_CFLAGS) \
   $(curl_CFLAGS)
@@ -20,6 +21,7 @@ lib_stream  =  {
LIBS = $(SOCKET_LIBS) \
   $(zlib_LIBS) \
   $(BZ_LIBS) \
+  $(snappy_LIBS) \
   $(liblzma_LIBS) \
   $(openssl_LIBS) \
   $(curl_LIBS) \
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -3567,6 +3567,8 @@ buffer_wastream(buffer *b, const char *n
return s;
 }
 
+
+
 /* -- */
 
 /* A buffered stream consists of a sequence of blocks.  Each block
@@ -3885,6 +3887,20 @@ bs_isalive(stream *ss)
 }
 
 static void
+bs_close(stream *ss)
+{
+   bs *s;
+
+   s = (bs *) ss->stream_data.p;
+   assert(s);
+   if (s == NULL)
+   return;
+   assert(s->s);
+   if (s->s)
+

MonetDB: protocol - Additional changes to byte_stream.

2016-08-04 Thread Hannes Muehleisen

Changeset: b60c634a07cd for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b60c634a07cd
Modified Files:
clients/mapilib/mapi.c
common/stream/stream.c
common/stream/stream.h
Branch: protocol
Log Message:

Additional changes to byte_stream.


diffs (truncated from 483 to 300 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2781,15 +2781,13 @@ mapi_reconnect(Mapi mid)
check_stream(mid, mid->to, "Could not send initial byte sequence", 
"mapi_reconnect", mid->error);
 
if (prot_version == prot10 || prot_version == prot10compressed) {
+   bstream *bs_to = (bstream*) mid->to;
+   bstream *bs_from = (bstream*) mid->from;
+
printf("Using protocol version %s.\n", prot_version == prot10  
? "PROT10" : "PROT10COMPRESSED");
-   // FIXME: destroy block streams and replace with appropriate 
streams
-#if 0
assert(isa_block_stream(mid->to));
assert(isa_block_stream(mid->from));
 
-   bs *bs_to = (bs*) mid->to;
-   bs *bs_from = (bs*) mid->from;
-
if (prot_version == prot10compressed) {
 #ifdef HAVE_LIBSNAPPY
mid->to = compressed_stream(bs_to->s, 
COMPRESSION_SNAPPY);
@@ -2798,13 +2796,16 @@ mapi_reconnect(Mapi mid)
assert(0);
 #endif
} else {
-   mid->to = byte_stream((bs_to->s);
-   mid->from = byte_stream(bs_from->s);
+   // FIXME: figure out proper stream sizes
+   mid->to = byte_stream(bs_to->s, 1024000);
+   mid->from = byte_stream(bs_from->s, 1024000);
+
}
-
-   close_stream(bs_to);
-   close_stream(bs_from);
-#endif
+   bs_to->s = NULL;
+   bs_from->s = NULL;
+   close_stream((stream*) bs_to);
+   close_stream((stream*) bs_from);
+
}
 
/* consume the welcome message from the server */
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -3885,20 +3885,6 @@ bs_isalive(stream *ss)
 }
 
 static void
-bs_close(stream *ss)
-{
-   bs *s;
-
-   s = (bs *) ss->stream_data.p;
-   assert(s);
-   if (s == NULL)
-   return;
-   assert(s->s);
-   if (s->s)
-   s->s->close(s->s);
-}
-
-static void
 bs_destroy(stream *ss)
 {
bs *s;
@@ -3914,6 +3900,24 @@ bs_destroy(stream *ss)
destroy(ss);
 }
 
+
+
+static void
+bs_close(stream *ss)
+{
+   bs *s;
+
+   s = (bs *) ss->stream_data.p;
+   assert(s);
+   if (s == NULL)
+   return;
+   assert(s->s);
+   if (s->s)
+   s->s->close(s->s);
+   bs_destroy(ss);
+}
+
+
 static void
 bs_clrerr(stream *s)
 {
@@ -3984,8 +3988,9 @@ bytestream_create(stream *s, size_t bufs
return NULL;
ns->s = s;
ns->bufsize = bufsize;
-   ns->bufpos = 0;
+   ns->bufpos = BYTESTREAM_OVERHEAD;
ns->buf = malloc(bufsize);
+   ns->bufend = ns->bufpos;
if (ns->buf == NULL) {
free(ns);
return NULL;
@@ -4003,20 +4008,17 @@ bytestream_flush(stream *ss)
return -1;
assert(ss->access == ST_WRITE);
assert(s->bufpos < s->bufsize);
-   if (ss->access == ST_WRITE) {
-   if (s->bufpos + BYTESTREAM_OVERHEAD <= s->bufsize) {
-   // if there is room in the buffer, end the buffer with 
a 0 so the reader knows that the stream has ended
-   lng data = 0;
-   memcpy(s->buf + s->bufpos, , sizeof(lng));
-   s->bufpos += sizeof(lng);
-   }
-   // FIXME: for compressed stream we can compress the buffer here 
before writing it
-   if (!s->s->write(s->s, s->buf, 1, s->bufpos)) {
-   ss->errnr = MNSTR_WRITE_ERROR;
-   return -1;
-   }
-   s->bufpos = 0;
+
+   memcpy(s->buf, >bufpos, sizeof(size_t));
+
+
+   // FIXME: for compressed stream we can compress the buffer here before 
writing it
+   if (!s->s->write(s->s, s->buf, 1, s->bufpos)) {
+   ss->errnr = MNSTR_WRITE_ERROR;
+   return -1;
}
+   s->bufpos = BYTESTREAM_OVERHEAD;
+
return 0;
 }
 
@@ -4031,227 +4033,125 @@ bytestream_write(stream *ss, const void 
return -1;
assert(ss->access == ST_WRITE);
assert(s->bufpos < s->bufsize);
-   if (todo + BYTESTREAM_OVERHEAD > s->bufsize)
+   if (todo > s->bufsize)
fprintf(stderr, "Content too big for buffer!\n");
ss->errnr = MNSTR_WRITE_ERROR;
return -1; // content does not fit into buffer

MonetDB: protocol - new branch for protocol rewrite

2016-08-02 Thread Hannes Muehleisen

Changeset: eb1b9781c58f for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=eb1b9781c58f
Modified Files:
clients/mapilib/mapi.c
common/stream/stream.c
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/sql_result.c
Branch: protocol
Log Message:

new branch for protocol rewrite


diffs (79 lines):

diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -2535,6 +2535,8 @@ mapi_reconnect(Mapi mid)
 
mid->connected = 1;
 
+   // FIXME: these need to become snappy streams (framing2)
+
if (!isa_block_stream(mid->to)) {
mid->to = block_stream(mid->to);
check_stream(mid, mid->to, mnstr_error(mid->to), 
"mapi_reconnect", mid->error);
@@ -3930,6 +3932,7 @@ read_into_cache(MapiHdl hdl, int lookahe
break;
case '%':
case '#':
+   //FIXME: add new result set parsing
case '&':
if (lookahead < 0)
lookahead = 1;
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -3921,6 +3921,11 @@ bs_clrerr(stream *s)
mnstr_clearerr(((bs *) s->stream_data.p)->s);
 }
 
+
+// FIXME: patch bs_read/bs_write etc
+// 10 MB max buffer size or so
+// unsigned int length or so
+
 stream *
 block_stream(stream *s)
 {
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -114,6 +114,8 @@ doChallenge(void *data)
char *buf = (char *) GDKmalloc(BLOCK + 1);
char challenge[13];
char *algos;
+
+   // FIXME: make this a snappy stream as well
stream *fdin = block_stream(((struct challengedata *) data)->in);
stream *fdout = block_stream(((struct challengedata *) data)->out);
bstream *bs;
@@ -142,8 +144,10 @@ doChallenge(void *data)
/* generate the challenge string */
generateChallenge(challenge, 8, 12);
algos = mcrypt_getHashAlgorithms();
+   // FIXME: add the newproto flag to algos and rename to 'capabilities' 
to hide the crime
+
/* note that we claim to speak proto 9 here for hashed passwords */
-   mnstr_printf(fdout, "%s:mserver:9:%s:%s:%s:",
+   mnstr_printf(fdout, "%s:mserver:10:%s:%s:%s:",
challenge,
algos,
 #ifdef WORDS_BIGENDIAN
@@ -171,6 +175,7 @@ doChallenge(void *data)
mnstr_printf(cntxt->fdout, "#SERVERlisten:client accepted\n");
mnstr_printf(cntxt->fdout, "#SERVERlisten:client string %s\n", buf);
 #endif
+   // FIXME: how can we avoid this
bs = bstream_create(fdin, 128 * BLOCK);
 
if (bs == NULL){
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -1313,6 +1313,8 @@ mvc_export_row(backend *b, stream *s, re
return (ok) ? 0 : -1;
 }
 
+
+//FIXME: rewrite this into something new
 static int
 mvc_export_table(backend *b, stream *s, res_table *t, BAT *order, BUN offset, 
BUN nr, char *btag, char *sep, char *rsep, char *ssep, char *ns)
 {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: pythonloader - xEy integer handling improved

2016-07-25 Thread Hannes Muehleisen

Changeset: ddfd4a5e4829 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ddfd4a5e4829
Modified Files:
gdk/gdk_atoms.c
Branch: pythonloader
Log Message:

xEy integer handling improved


diffs (67 lines):

diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -578,23 +578,6 @@ batWrite(const bat *a, stream *s, size_t
return mnstr_writeIntArray(s, (const int *) a, cnt) ? GDK_SUCCEED : 
GDK_FAIL;
 }
 
-#ifdef HAVE_HGE
-static hge h_pow(hge base, hge exp) {
-   hge result = 1;
-#else
-static lng h_pow(lng base, lng exp) {
-   lng result = 1;
-#endif
-while (exp)
-{
-if (exp & 1)
-result *= base;
-exp >>= 1;
-base *= base;
-}
-return result;
-}
-
 
 /*
  * numFromStr parses the head of the string for a number, accepting an
@@ -656,19 +639,34 @@ numFromStr(const char *src, int *len, vo
}
base = 10 * base + base10(*p);
p++;
+   /* Special case: xEy = x*10^y handling part 1 */
if (*p == 'E' || *p == 'e') {
+   // if there is a second E in the string we give up
+   if (expbase > -1) {
+   memcpy(*dst, ATOMnilptr(tp), sz);
+   return 0;
+   }
expbase = base;
base = 0;
p++;
}
} while (num10(*p));
+   /* Special case: xEy = x*10^y handling part 2 */
if (expbase > -1) {
-   dbl checkval = fabs(((dbl) expbase) * (dbl) pow(10, (dbl) 
base));
-   if (checkval >= maxdiv10 * 10) {
-   memcpy(*dst, ATOMnilptr(tp), sz);
-   return 0;
+#ifdef HAVE_HGE
+   hge res = expbase;
+#else
+   lng res = expbase;
+#endif
+   while (base > 0) {
+   if (res > maxdiv10) {
+   memcpy(*dst, ATOMnilptr(tp), sz);
+   return 0;
+   }
+   res *= 10L;
+   base--;
}
-   base = expbase * h_pow(10, base);
+   base = res;
}
base *= sign;
 
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - new tests

2016-07-19 Thread Hannes Muehleisen

Changeset: dc8dad5e60d3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dc8dad5e60d3
Added Files:
sql/test/malloc_fail/Tests/setmemorylimit-fail.sql
sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err
sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out
sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql
Branch: default
Log Message:

new tests


diffs (98 lines):

diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql 
b/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.sql
@@ -0,0 +1,3 @@
+create procedure setmemorylimit(nbytes BIGINT) external name 
"io"."setmemorylimit";
+call setmemorylimit(1000);
+select name from tables where 1=0;
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err 
b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.err
@@ -0,0 +1,40 @@
+stderr of test 'setmemorylimit-fail` in directory 'sql/test/malloc_fail` 
itself:
+
+
+# 15:41:50 >  
+# 15:41:50 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38119" "--set" 
"mapi_usock=/var/tmp/mtest-56431/.s.monetdb.38119" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail"
+# 15:41:50 >  
+
+# builtin opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 38119
+# cmdline opt  mapi_usock = /var/tmp/mtest-56431/.s.monetdb.38119
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail
+# cmdline opt  gdk_debug = 536870922
+
+# 15:41:50 >  
+# 15:41:50 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-56431" "--port=38119"
+# 15:41:50 >  
+
+MAPI  = (monetdb) /var/tmp/mtest-56431/.s.monetdb.38119
+QUERY = create procedure setmemorylimit(nbytes BIGINT) external name 
"io"."setmemorylimit";
+ERROR = !CREATE PROCEDURE: name 'setmemorylimit' (bigint(64)) already in use
+MAPI  = (monetdb) /var/tmp/mtest-56431/.s.monetdb.38119
+QUERY = select name from tables where 1=0;
+ERROR = !Could not create SQL allocator
+
+# 15:41:50 >  
+# 15:41:50 >  "Done."
+# 15:41:50 >  
+
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out 
b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail.stable.out
@@ -0,0 +1,32 @@
+stdout of test 'setmemorylimit-fail` in directory 'sql/test/malloc_fail` 
itself:
+
+
+# 15:41:50 >  
+# 15:41:50 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=38119" "--set" 
"mapi_usock=/var/tmp/mtest-56431/.s.monetdb.38119" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail"
+# 15:41:50 >  
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test_malloc_fail', using 4 threads
+# Compiled for x86_64-apple-darwin15.5.0/64bit with 128bit integers
+# Found 16.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:38119/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-56431/.s.monetdb.38119
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+
+Ready.
+!Memory allocation failed.
+
+# 15:41:50 >  
+# 15:41:50 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-56431" "--port=38119"
+# 15:41:50 >  
+
+
+# 15:41:50 >  
+# 15:41:50 >  "Done."
+# 15:41:50 >  
+
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql 
b/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit-fail2.sql
@@ -0,0 +1,3 @@
+create procedure setmemorylimit(nbytes BIGINT) external name 
"io"."setmemorylimit";
+call setmemorylimit(0);
+select name from tables where 1=0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - changed semantics of memory limit to only fai...

2016-07-19 Thread Hannes Muehleisen

Changeset: 95fbd5f0b4a0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=95fbd5f0b4a0
Modified Files:
gdk/gdk_utils.c
gdk/gdk_utils.h
monetdb5/mal/mal_instruction.c
monetdb5/modules/mal/mal_mapi.c
sql/backends/monet5/sql_scenario.c
sql/common/sql_list.c
sql/common/sql_mem.c
sql/server/rel_semantic.c
sql/storage/sql_catalog.c
sql/test/malloc_fail/Tests/All
sql/test/malloc_fail/Tests/setmemorylimit.sql
Branch: default
Log Message:

changed semantics of memory limit to only fail allocations larger than limit


diffs (158 lines):

diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -310,7 +310,7 @@ int GDK_vm_trim = 1;
  * fall-back for other compilers. */
 #include "gdk_atomic.h"
 static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0;
-static volatile ATOMIC_TYPE GDK_mallocedbytes_limit = 0;
+static volatile lng GDK_mallocedbytes_limit = -1;
 static volatile ATOMIC_TYPE GDK_vm_cursize = 0;
 #ifdef GDK_VM_KEEPHISTO
 volatile ATOMIC_TYPE GDK_vm_nallocs[MAX_BIT] = { 0 };
@@ -1682,7 +1682,7 @@ GDKmalloc_prefixsize(size_t size)
return s;
 }
 
-gdk_export void GDKsetmemorylimit(size_t nbytes) {
+gdk_export void GDKsetmemorylimit(lng nbytes) {
GDK_mallocedbytes_limit = nbytes;
 }
 
@@ -1706,8 +1706,7 @@ GDKmallocmax(size_t size, size_t *maxsiz
}
 #ifndef NDEBUG
/* fail malloc for testing purposes depending on set limit */
-   if (GDK_mallocedbytes_limit > 0 &&
-   (GDK_mallocedbytes_estimate + size + 
MALLOC_EXTRA_SPACE) > GDK_mallocedbytes_limit) {
+   if (GDK_mallocedbytes_limit >= 0 && size >(size_t) 
GDK_mallocedbytes_limit) {
return NULL;
}
 #endif
diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h
--- a/gdk/gdk_utils.h
+++ b/gdk/gdk_utils.h
@@ -81,7 +81,7 @@ gdk_export void MT_init(void);/*  init 
 gdk_export int GDKinit(opt *set, int setlen);
 
 /* used for testing only */
-gdk_export void GDKsetmemorylimit(size_t nbytes);
+gdk_export void GDKsetmemorylimit(lng nbytes);
 
 /*
  * Upon closing the session, all persistent BATs should be saved and
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -1543,6 +1543,8 @@ pushEndInstruction(MalBlkPtr mb)
 
p = newInstruction(mb, ENDsymbol);
if (!p) {
+   mb->errors++;
+   showException(GDKout, MAL, "pushEndInstruction", "failed to 
create instruction (out of memory?)");
return;
}
p->argc = 0;
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -367,8 +367,9 @@ SERVERlistenThread(SOCKET *Sock)
data = GDKmalloc(sizeof(*data));
if (!data) {
closesocket(msgsock);
-   msg = "memory allocation failed";
-   goto error;
+   showException(GDKstdout, MAL, "initClient",
+ "cannot allocate memory");
+   continue;
}
data->in = socket_rastream(msgsock, "Server read");
data->out = socket_wastream(msgsock, "Server write");
diff --git a/sql/backends/monet5/sql_scenario.c 
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -936,6 +936,12 @@ SQLparser(Client c)
 * this point if this is a recursive call. */
if (!m->sa)
m->sa = sa_create();
+   if (!m->sa) {
+   mnstr_printf(out, "!Could not create SQL allocator\n");
+   mnstr_flush(out);
+   c->mode = FINISHCLIENT;
+   throw(SQL, "SQLparser", "Could not create SQL allocator");
+   }
 
m->emode = m_normal;
m->emod = mod_none;
diff --git a/sql/common/sql_list.c b/sql/common/sql_list.c
--- a/sql/common/sql_list.c
+++ b/sql/common/sql_list.c
@@ -24,6 +24,9 @@ list *
 list_create(fdestroy destroy)
 {
list *l = MNEW(list);
+   if (!l) {
+   return NULL;
+   }
 
l->sa = NULL;
l->destroy = destroy;
diff --git a/sql/common/sql_mem.c b/sql/common/sql_mem.c
--- a/sql/common/sql_mem.c
+++ b/sql/common/sql_mem.c
@@ -44,7 +44,9 @@ sql_ref_dec(sql_ref *r)
 sql_allocator *sa_create(void)
 {
sql_allocator *sa = MNEW(sql_allocator);
-   
+   if (!sa) {
+   return NULL;
+   }
sa->size = 64;
sa->nr = 1;
sa->blks = NEW_ARRAY(char*,sa->size);
diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c
--- a/sql/server/rel_semantic.c
+++ b/sql/server/rel_semantic.c
@@ -43,6 +43,9 @@ rel_parse(mvc *m, sql_schema *s, char *q
 
b =

MonetDB: default - newline

2016-07-19 Thread Hannes Muehleisen

Changeset: d23c1aef84ae for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d23c1aef84ae
Modified Files:
monetdb5/modules/mal/mal_io.mal
Branch: default
Log Message:

newline


diffs (10 lines):

diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal
--- a/monetdb5/modules/mal/mal_io.mal
+++ b/monetdb5/modules/mal/mal_io.mal
@@ -64,4 +64,4 @@ comment "Import a BAT from an ASCII dump
 # used for testing malloc failures
 command setmemorylimit(nbytes:lng):void
 address IOsetmemorylimit
-comment "Set memory limit for testing";
\ No newline at end of file
+comment "Set memory limit for testing";
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - first test case using memory allocation limit...

2016-07-19 Thread Hannes Muehleisen

Changeset: 946681f654d8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=946681f654d8
Added Files:
sql/test/malloc_fail/Tests/All
sql/test/malloc_fail/Tests/setmemorylimit.sql
sql/test/malloc_fail/Tests/setmemorylimit.stable.err
sql/test/malloc_fail/Tests/setmemorylimit.stable.out
Branch: default
Log Message:

first test case using memory allocation limit, for now all is well


diffs (137 lines):

diff --git a/sql/test/malloc_fail/Tests/All b/sql/test/malloc_fail/Tests/All
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/All
@@ -0,0 +1,1 @@
+setmemorylimit
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.sql 
b/sql/test/malloc_fail/Tests/setmemorylimit.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit.sql
@@ -0,0 +1,7 @@
+create procedure setmemorylimit(nbytes BIGINT) external name 
"io"."setmemorylimit";
+call setmemorylimit(0);
+call setmemorylimit(1);
+-- this should work fine
+select name from tables where 1=0;
+call setmemorylimit(0);
+drop procedure setmemorylimit;
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.stable.err 
b/sql/test/malloc_fail/Tests/setmemorylimit.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit.stable.err
@@ -0,0 +1,34 @@
+stderr of test 'setmemorylimit` in directory 'sql/test/malloc_fail` itself:
+
+
+# 11:39:58 >  
+# 11:39:58 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=36345" "--set" 
"mapi_usock=/var/tmp/mtest-21164/.s.monetdb.36345" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail"
+# 11:39:58 >  
+
+# builtin opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 36345
+# cmdline opt  mapi_usock = /var/tmp/mtest-21164/.s.monetdb.36345
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = 
/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail
+# cmdline opt  gdk_debug = 536870922
+
+# 11:39:58 >  
+# 11:39:58 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-21164" "--port=36345"
+# 11:39:58 >  
+
+
+# 11:39:59 >  
+# 11:39:59 >  "Done."
+# 11:39:59 >  
+
diff --git a/sql/test/malloc_fail/Tests/setmemorylimit.stable.out 
b/sql/test/malloc_fail/Tests/setmemorylimit.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/malloc_fail/Tests/setmemorylimit.stable.out
@@ -0,0 +1,75 @@
+stdout of test 'setmemorylimit` in directory 'sql/test/malloc_fail` itself:
+
+
+# 11:39:58 >  
+# 11:39:58 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=36345" "--set" 
"mapi_usock=/var/tmp/mtest-21164/.s.monetdb.36345" "--set" "monet_prompt=" 
"--forcemito" 
"--dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_sql_test_malloc_fail"
+# 11:39:58 >  
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_sql_test_malloc_fail', using 4 threads
+# Compiled for x86_64-apple-darwin15.5.0/64bit with 128bit integers
+# Found 16.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:36345/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-21164/.s.monetdb.36345
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 18_index.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 24_zorder.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 39_analytics_hge.sql
+# loading sql script: 40_geom.sql
+# loading sql script: 40_json.sql
+# loading sql script: 40_json_hge.sql
+# loading sql script: 41_md5sum.sql
+# loading sql script: 45_uuid.sql
+# loading

MonetDB: default - fixed crash on malloc failure

2016-07-19 Thread Hannes Muehleisen

Changeset: 9c5b7d0a3f58 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9c5b7d0a3f58
Modified Files:
monetdb5/mal/mal_exception.c
monetdb5/mal/mal_instruction.c
Branch: default
Log Message:

fixed crash on malloc failure


diffs (39 lines):

diff --git a/monetdb5/mal/mal_exception.c b/monetdb5/mal/mal_exception.c
--- a/monetdb5/mal/mal_exception.c
+++ b/monetdb5/mal/mal_exception.c
@@ -141,7 +141,9 @@ showException(stream *out, enum malexcep
va_end(ap);
 
dumpExceptionsToStream(out, msg);
-   GDKfree(msg);
+   if (msg && msg != M5OutOfMemory) {
+   GDKfree(msg);
+   }
 }
 
 /**
diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -410,8 +410,10 @@ newInstruction(MalBlkPtr mb, int kind)
}
if (p == NULL) {
p = GDKzalloc(MAXARG * sizeof(p->argv[0]) + 
offsetof(InstrRecord, argv));
-   if (p == NULL)
+   if (p == NULL) {
+   showException(GDKout, MAL, "pushEndInstruction", 
"memory allocation failure");
return NULL;
+   }
p->maxarg = MAXARG;
}
p->typechk = TYPE_UNKNOWN;
@@ -1540,6 +1542,9 @@ pushEndInstruction(MalBlkPtr mb)
InstrPtr p;
 
p = newInstruction(mb, ENDsymbol);
+   if (!p) {
+   return;
+   }
p->argc = 0;
p->retc = 0;
p->argv[0] = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - fix compiler warning with clang

2016-07-19 Thread Hannes Muehleisen

Changeset: 9f7fbbf9727c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=9f7fbbf9727c
Modified Files:
monetdb5/mal/mal_instruction.c
Branch: default
Log Message:

fix compiler warning with clang


diffs (12 lines):

diff --git a/monetdb5/mal/mal_instruction.c b/monetdb5/mal/mal_instruction.c
--- a/monetdb5/mal/mal_instruction.c
+++ b/monetdb5/mal/mal_instruction.c
@@ -600,7 +600,7 @@ findVariableLength(MalBlkPtr mb, str nam
int j;
 
for (i = mb->vtop - 1; i >= 0; i--)
-   if (mb->var[i] && mb->var[i]->id) {
+   if (mb->var[i]) { /* mb->var[i]->id will always evaluate to 
true */
str s = mb->var[i]->id;
 
j = 0;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - method to set a memory limit after which mall...

2016-07-19 Thread Hannes Muehleisen

Changeset: 1f5a6fd13a28 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1f5a6fd13a28
Modified Files:
gdk/gdk_utils.c
gdk/gdk_utils.h
monetdb5/modules/mal/mal_io.c
monetdb5/modules/mal/mal_io.h
monetdb5/modules/mal/mal_io.mal
Branch: default
Log Message:

method to set a memory limit after which malloc fails for testing purposes


diffs (97 lines):

diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -310,6 +310,7 @@ int GDK_vm_trim = 1;
  * fall-back for other compilers. */
 #include "gdk_atomic.h"
 static volatile ATOMIC_TYPE GDK_mallocedbytes_estimate = 0;
+static volatile ATOMIC_TYPE GDK_mallocedbytes_limit = 0;
 static volatile ATOMIC_TYPE GDK_vm_cursize = 0;
 #ifdef GDK_VM_KEEPHISTO
 volatile ATOMIC_TYPE GDK_vm_nallocs[MAX_BIT] = { 0 };
@@ -1681,6 +1682,11 @@ GDKmalloc_prefixsize(size_t size)
return s;
 }
 
+gdk_export void GDKsetmemorylimit(size_t nbytes) {
+   GDK_mallocedbytes_limit = nbytes;
+}
+
+
 /*
  * The emergency flag can be set to force a fatal error if needed.
  * Otherwise, the caller is able to deal with the lack of memory.
@@ -1698,6 +1704,13 @@ GDKmallocmax(size_t size, size_t *maxsiz
GDKfatal("GDKmallocmax: called with size " SZFMT "", size);
 #endif
}
+#ifndef NDEBUG
+   /* fail malloc for testing purposes depending on set limit */
+   if (GDK_mallocedbytes_limit > 0 &&
+   (GDK_mallocedbytes_estimate + size + 
MALLOC_EXTRA_SPACE) > GDK_mallocedbytes_limit) {
+   return NULL;
+   }
+#endif
size = (size + 7) & ~7; /* round up to a multiple of eight */
s = GDKmalloc_prefixsize(size);
if (s == NULL) {
@@ -1710,6 +1723,7 @@ GDKmallocmax(size_t size, size_t *maxsiz
}
GDKfatal("GDKmallocmax: failed for " SZFMT " bytes", 
size);
} else {
+   /* TODO why are we printing this on stderr? */
fprintf(stderr, "#GDKmallocmax: recovery ok. 
Continuing..\n");
}
}
diff --git a/gdk/gdk_utils.h b/gdk/gdk_utils.h
--- a/gdk/gdk_utils.h
+++ b/gdk/gdk_utils.h
@@ -80,6 +80,9 @@ gdk_export size_t _MT_pagesize;
 gdk_export void MT_init(void); /*  init the package. */
 gdk_export int GDKinit(opt *set, int setlen);
 
+/* used for testing only */
+gdk_export void GDKsetmemorylimit(size_t nbytes);
+
 /*
  * Upon closing the session, all persistent BATs should be saved and
  * the transient BATs should be removed.  The buffer pool manager
diff --git a/monetdb5/modules/mal/mal_io.c b/monetdb5/modules/mal/mal_io.c
--- a/monetdb5/modules/mal/mal_io.c
+++ b/monetdb5/modules/mal/mal_io.c
@@ -776,3 +776,11 @@ IOimport(void *ret, bat *bid, str *fnme)
return MAL_SUCCEED;
 }
 
+
+
+str
+IOsetmemorylimit(void *res, lng *nbytes) {
+   (void) res;
+   GDKsetmemorylimit(*nbytes);
+   return MAL_SUCCEED;
+}
diff --git a/monetdb5/modules/mal/mal_io.h b/monetdb5/modules/mal/mal_io.h
--- a/monetdb5/modules/mal/mal_io.h
+++ b/monetdb5/modules/mal/mal_io.h
@@ -28,4 +28,7 @@ mal_export str IOimport(void *ret, bat *
 mal_export str io_stdin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
 mal_export str io_stdout(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
 mal_export str io_stderr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr 
pci);
+
+mal_export str IOsetmemorylimit(void *res, lng *nbytes);
+
 #endif /* _PRINT_H_ */
diff --git a/monetdb5/modules/mal/mal_io.mal b/monetdb5/modules/mal/mal_io.mal
--- a/monetdb5/modules/mal/mal_io.mal
+++ b/monetdb5/modules/mal/mal_io.mal
@@ -60,3 +60,8 @@ address IOimport
 comment "Import a BAT from an ASCII dump. The tuples are appended to the
  first argument. Its signature must match the dump,
  else parsing errors will occur as an exception.";
+
+# used for testing malloc failures
+command setmemorylimit(nbytes:lng):void
+address IOsetmemorylimit
+comment "Set memory limit for testing";
\ No newline at end of file
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - fixed crash in mapi code

2016-07-19 Thread Hannes Muehleisen

Changeset: 0c5dff585dd5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0c5dff585dd5
Modified Files:
monetdb5/modules/mal/mal_mapi.c
Branch: default
Log Message:

fixed crash in mapi code


diffs (15 lines):

diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -365,6 +365,11 @@ SERVERlistenThread(SOCKET *Sock)
fflush(stdout);
 #endif
data = GDKmalloc(sizeof(*data));
+   if (!data) {
+   closesocket(msgsock);
+   msg = "memory allocation failed";
+   goto error;
+   }
data->in = socket_rastream(msgsock, "Server read");
data->out = socket_wastream(msgsock, "Server write");
if (MT_create_thread(, doChallenge, data, MT_THR_JOINABLE)) 
{
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: pythonloader - directly creating output tables for crea...

2016-06-20 Thread Hannes Muehleisen

Changeset: 751e80b51d1c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=751e80b51d1c
Added Files:
sql/backends/monet5/Tests/pyloader07.sql
sql/backends/monet5/Tests/pyloader07.stable.err
sql/backends/monet5/Tests/pyloader07.stable.out
Modified Files:
monetdb5/extras/pyapi/connection.c
sql/backends/monet5/Tests/All
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
Branch: pythonloader
Log Message:

directly creating output tables for create table from loader


diffs (truncated from 373 to 300 lines):

diff --git a/monetdb5/extras/pyapi/connection.c 
b/monetdb5/extras/pyapi/connection.c
--- a/monetdb5/extras/pyapi/connection.c
+++ b/monetdb5/extras/pyapi/connection.c
@@ -244,106 +244,76 @@ str _connection_query(Client cntxt, char
 return res;
 }
 
-static str _connection_append_table(Client cntxt, char *sname, char *tname, 
EmitCol *columns, size_t ncols) {
+
+str _connection_create_table(Client cntxt, char *sname, char *tname, EmitCol 
*columns, size_t ncols) {
 size_t i;
-size_t nvar = 6; // variables we need to make up
-MalBlkRecord mb;
-MalStack* stk = NULL;
-InstrRecord*  pci = NULL;
-str res = MAL_SUCCEED;
-VarRecord bat_varrec;
-mvc* m = ((backend *) cntxt->sqlcontext)->mvc;
+sql_table *t;
+sql_schema *s;
+mvc *sql = NULL;
+str msg = MAL_SUCCEED;
 
-assert(tname != NULL && columns != NULL && ncols > 0);
+   if ((msg = getSQLContext(cntxt, NULL, , NULL)) != NULL)
+   return msg;
+   if ((msg = checkSQLContext(cntxt)) != NULL)
+   return msg;
 
-// very black MAL magic below
-mb.var = GDKmalloc(nvar * sizeof(VarRecord*));
-stk = GDKmalloc(sizeof(MalStack) + nvar * sizeof(ValRecord));
-pci = GDKmalloc(sizeof(InstrRecord) + nvar * sizeof(int));
-assert(mb.var != NULL && stk != NULL && pci != NULL); // cough, cough
-bat_varrec.type = TYPE_bat;
-for (i = 0; i < nvar; i++) {
-pci->argv[i] = i;
-}
-stk->stk[0].vtype = TYPE_int;
-stk->stk[2].val.sval = (str) sname;
-stk->stk[2].vtype = TYPE_str;
-stk->stk[3].val.sval = (str) tname;
-stk->stk[3].vtype = TYPE_str;
-stk->stk[4].vtype = TYPE_str;
-stk->stk[5].vtype = TYPE_bat;
-mb.var[5] = _varrec;
-for (i = 0; i < ncols; i++) {
-EmitCol col = columns[i];
-stk->stk[4].val.sval = col.name;
-stk->stk[5].val.bval = col.b->batCacheid;
+   /* for some reason we don't have an allocator here so make one */
+   sql->sa = sa_create();
 
-res = (*mvc_append_wrap_ptr)(cntxt, , stk, pci);
-if (res != NULL) {
-break;
+if (!sname) sname = "sys";
+   if (!(s = mvc_bind_schema(sql, sname))) {
+   msg = sql_error(sql, 02, "3F000!CREATE TABLE: no such schema 
'%s'", sname);
+   goto cleanup;
+   }
+   if (!(t = mvc_create_table(sql, s, tname, tt_table, 0, 
SQL_DECLARED_TABLE, CA_COMMIT, -1))) {
+   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not create 
table '%s'", tname);
+   goto cleanup;
+   }
+
+for(i = 0; i < ncols; i++) {
+BAT *b = columns[i].b;
+sql_subtype *tpe = sql_bind_localtype(ATOMname(b->T->type));
+sql_column *col = NULL;
+
+if (!tpe) {
+   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not find 
type for column");
+   goto cleanup;
+}
+
+col = mvc_create_column(sql, t, columns[i].name, tpe);
+if (!col) {
+   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not create 
column %s", columns[i].name);
+   goto cleanup;
 }
 }
-if (res == MAL_SUCCEED) {
-(*sqlcleanup_ptr)(m, 0);
+msg = create_table_or_view(sql, sname, t, 0);
+if (msg != MAL_SUCCEED) {
+   goto cleanup;
 }
-GDKfree(mb.var);
-GDKfree(stk);
-GDKfree(pci);
-return res;
-}
+t = mvc_bind_table(sql, s, tname);
+if (!t) {
+   msg = sql_error(sql, 02, "3F000!CREATE TABLE: could not bind 
table %s", tname);
+   goto cleanup;
+}
+for(i = 0; i < ncols; i++) {
+BAT *b = columns[i].b;
+sql_column *col = NULL;
 
-static char *BatType_ToSQLType(int type) {
-switch (type) {
-case TYPE_bit:
-case TYPE_bte: return "TINYINT";
-case TYPE_sht: return "SMALLINT";
-case TYPE_int: return "INTEGER";
-case TYPE_lng: return "BIGINT";
-case TYPE_flt: return "FLOAT";
-case TYPE_dbl: return "DOUBLE";
-case TYPE_str: return "STRING";
-case TYPE_hge: return "HUGEINT";
-case TYPE_oid: return "UNKNOWN";
-default: return "UNKNOWN";
-}
-}
-
-str _connection_create_table(Client cntxt, char *sname, char *tname, EmitCol 
*columns, size_t ncols) {
-char *query = NULL;
-size_t i;
-size_t query_size = 255, query_index = 0;
-res_table

MonetDB: pythonloader - second stage COPY INTO sometable FROM LO...

2016-06-15 Thread Hannes Muehleisen

Changeset: 90f85fc4830a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90f85fc4830a
Added Files:
monetdb5/extras/pyapi/emit.c
monetdb5/extras/pyapi/emit.h
monetdb5/extras/pyapi/pyloader.c
sql/backends/monet5/Tests/pyapi30.stable.err
sql/backends/monet5/Tests/pyapi30.stable.out
sql/backends/monet5/Tests/pyapi31.sql
sql/backends/monet5/Tests/pyapi31.stable.err
sql/backends/monet5/Tests/pyapi31.stable.out
Modified Files:
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/formatinput.h
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
monetdb5/extras/rapi/rapi.c
sql/backends/monet5/Tests/All
sql/backends/monet5/Tests/pyapi30.sql
sql/backends/monet5/sql_gencode.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_mvc.h
sql/server/sql_parser.y
Branch: pythonloader
Log Message:

second stage COPY INTO sometable FROM LOADER somepythonfunction(); basic 
functionality is there


diffs (truncated from 1512 to 300 lines):

diff --git a/monetdb5/extras/pyapi/Makefile.ag 
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -24,7 +24,7 @@ MTSAFE
 lib__pyapi = {
MODULE
DIR = libdir/monetdb5
-   SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h 
type_conversion.c type_conversion.h  formatinput.c formatinput.h connection.c 
connection.h unspecified_evil.h
+   SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h 
type_conversion.c type_conversion.h  formatinput.c formatinput.h connection.c 
connection.h unspecified_evil.h pyloader.c emit.h emit.c
XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
   ../../../gdk/libbat \
diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/pyapi/emit.c
@@ -0,0 +1,195 @@
+
+#include "emit.h"
+#include "type_conversion.h"
+#include "interprocess.h"
+
+#if PY_MAJOR_VERSION >= 3
+#define IS_PY3K
+#define PyString_CheckExact PyUnicode_CheckExact
+#define PyString_FromString PyUnicode_FromString
+#endif
+
+#define scalar_convert(tpe) {\
+   tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(, 42, 
); \
+   BUNappend(self->cols[i].b, , 0); \
+   if (msg != MAL_SUCCEED) { \
+   PyErr_SetString(PyExc_TypeError, "conversion failed"); /* TODO: 
better error message */ \
+   return NULL; \
+   }}
+
+static PyObject *
+_emit_emit(Py_EmitObject *self, PyObject *args) {
+   size_t i, ai;
+   ssize_t el_count = -1;
+   str msg = MAL_SUCCEED;
+   (void) self;
+   if (!PyDict_Check(args)) {
+   // complain
+   PyErr_SetString(PyExc_TypeError, "need dict");
+   return NULL;
+   }
+
+   for (i = 0; i < self->ncols; i++) {
+   PyObject *dictEntry = PyDict_GetItemString(args, 
self->cols[i].name);
+   ssize_t this_size = 1;
+   if (dictEntry) {
+   if (!PyType_IsPyScalar(dictEntry)) {
+   this_size = Py_SIZE(dictEntry);
+   }
+   if (el_count < 0) el_count = this_size;
+   else {
+   if (el_count != this_size) {
+   PyErr_SetString(PyExc_TypeError, "need 
same length values");
+   // todo: better error message!
+   return NULL;
+   }
+   }
+   }
+   }
+   if (el_count < 1) {
+   PyErr_SetString(PyExc_TypeError, "need at least some values");
+   // todo: better error message!
+   return NULL;
+   }
+
+   // TODO: check for dict entries not matched by any column and complain 
if present
+
+
+   for (i = 0; i < self->ncols; i++) {
+   PyObject *dictEntry = PyDict_GetItemString(args, 
self->cols[i].name);
+   if (dictEntry) {
+   if (PyType_IsPyScalar(dictEntry)) {
+   switch (self->cols[i].b->T->type)
+   {
+   case TYPE_bit:
+   scalar_convert(bit);
+   break;
+   case TYPE_bte:
+   scalar_convert(bte);
+   break;
+   case TYPE_sht:
+   scalar_convert(sht);
+

MonetDB: pythonloader - Support for int syntax 4E6 (= 4000000) i...

2016-06-15 Thread Hannes Muehleisen

Changeset: 5d255bd86516 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5d255bd86516
Added Files:
sql/backends/monet5/Tests/int_notation_1e5.sql
sql/backends/monet5/Tests/int_notation_1e5.stable.err
sql/backends/monet5/Tests/int_notation_1e5.stable.out
Modified Files:
gdk/gdk_atoms.c
sql/backends/monet5/Tests/All
Branch: pythonloader
Log Message:

Support for int syntax 4E6 (= 400) in COPY INTO


diffs (228 lines):

diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c
--- a/gdk/gdk_atoms.c
+++ b/gdk/gdk_atoms.c
@@ -578,6 +578,24 @@ batWrite(const bat *a, stream *s, size_t
return mnstr_writeIntArray(s, (const int *) a, cnt) ? GDK_SUCCEED : 
GDK_FAIL;
 }
 
+#ifdef HAVE_HGE
+static hge h_pow(hge base, hge exp) {
+   hge result = 1;
+#else
+static lng h_pow(lng base, lng exp) {
+   lng result = 1;
+#endif
+while (exp)
+{
+if (exp & 1)
+result *= base;
+exp >>= 1;
+base *= base;
+}
+return result;
+}
+
+
 /*
  * numFromStr parses the head of the string for a number, accepting an
  * optional sign. The code has been prepared to continue parsing by
@@ -592,9 +610,11 @@ numFromStr(const char *src, int *len, vo
int sz = ATOMsize(tp);
 #ifdef HAVE_HGE
hge base = 0;
+   hge expbase = -1;
const hge maxdiv10 = GDK_hge_max / 10;
 #else
lng base = 0;
+   lng expbase = -1;
const lng maxdiv10 = LL_CONSTANT(922337203685477580); /*7*/
 #endif
const int maxmod10 = 7; /* max value % 10 */
@@ -636,8 +656,22 @@ numFromStr(const char *src, int *len, vo
}
base = 10 * base + base10(*p);
p++;
+   if (*p == 'E' || *p == 'e') {
+   expbase = base;
+   base = 0;
+   p++;
+   }
} while (num10(*p));
+   if (expbase > -1) {
+   dbl checkval = fabs(expbase * pow(10, base));
+   if (checkval >= maxdiv10 * 10) {
+   memcpy(*dst, ATOMnilptr(tp), sz);
+   return 0;
+   }
+   base = expbase * h_pow(10, base);
+   }
base *= sign;
+
switch (sz) {
case 1: {
bte **dstbte = (bte **) dst;
diff --git a/sql/backends/monet5/Tests/All b/sql/backends/monet5/Tests/All
--- a/sql/backends/monet5/Tests/All
+++ b/sql/backends/monet5/Tests/All
@@ -59,3 +59,5 @@ HAVE_LIBPY?pyapi29
 
 simpleSQL
 limithack
+
+HAVE_HUGEINT?int_notation_1e5
diff --git a/sql/backends/monet5/Tests/int_notation_1e5.sql 
b/sql/backends/monet5/Tests/int_notation_1e5.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/int_notation_1e5.sql
@@ -0,0 +1,13 @@
+START TRANSACTION;
+CREATE TABLE t1e4 (a HUGEINT, b HUGEINT);
+COPY 7 RECORDS INTO t1e4 FROM STDIN NULL AS '';
+01E4|1|
+1E4|1|
+1E5|10|
+-2E5|-20|
+-002E015|-2000|
+1E38|100|
+-42E36|-42|
+
+SELECT *, a-b AS diff FROM t1e4;
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/int_notation_1e5.stable.err 
b/sql/backends/monet5/Tests/int_notation_1e5.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/int_notation_1e5.stable.err
@@ -0,0 +1,36 @@
+stderr of test 'int_notation_1e5` in directory 'sql/backends/monet5` itself:
+
+
+# 11:11:20 >  
+# 11:11:20 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=37652" "--set" 
"mapi_usock=/var/tmp/mtest-6682/.s.monetdb.37652" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5" 
"--set" "embedded_r=yes" "--set" "embedded_py=true"
+# 11:11:20 >  
+
+# builtin opt  gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 37652
+# cmdline opt  mapi_usock = /var/tmp/mtest-6682/.s.monetdb.37652
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_sql_backends_monet5
+# cmdline opt  embedded_r = yes
+# cmdline opt  embedded_py = true
+# cmdline opt  gdk_debug = 536870922
+
+# 11:11:20 >  
+# 11:11:20 >  "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" 
"--host=/var/tmp/mtest-6682" "--port=37652"
+# 11:11:20 >  
+
+
+# 11:11:20 >  
+# 11:11:20 >  "Done."
+# 11:11:20 >  
+
diff --git a/sql/backends/monet5/Tests/int_notation_1e5.stable.out 
b/sql/backends/monet5/Tests/int_notation_1e5.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/int_notation_1e5.stable.out

MonetDB: pythonloader - test case

2016-06-14 Thread Hannes Muehleisen

Changeset: 84b713b6251e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=84b713b6251e
Added Files:
sql/backends/monet5/Tests/pyapi30.sql
Branch: pythonloader
Log Message:

test case


diffs (24 lines):

diff --git a/sql/backends/monet5/Tests/pyapi30.sql 
b/sql/backends/monet5/Tests/pyapi30.sql
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/pyapi30.sql
@@ -0,0 +1,19 @@
+
+START TRANSACTION;
+
+CREATE TABLE mytable(i DOUBLE, d STRING);
+
+CREATE LOADER myfunc() LANGUAGE PYTHON {
+   emit({'i':42,'d':'hello'})
+};
+
+SELECT * FROM functions WHERE name='myfunc';
+
+EXPLAIN COPY INTO mytable FROM LOADER myfunc();
+
+COPY INTO mytable FROM LOADER myfunc();
+
+DROP TABLE mytable;
+DROP LOADER myfunc;
+
+ROLLBACK;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: pythonloader - first stage COPY INTO sometable FROM LOA...

2016-06-14 Thread Hannes Muehleisen

Changeset: d38a839b1d0a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d38a839b1d0a
Modified Files:
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
sql/backends/monet5/sql_gencode.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/rel_select.c
sql/server/rel_semantic.c
sql/server/rel_updates.c
sql/server/sql_mvc.h
sql/server/sql_parser.h
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: pythonloader
Log Message:

first stage COPY INTO sometable FROM LOADER somepythonfunction();


diffs (truncated from 368 to 300 lines):

diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -418,6 +418,12 @@ PyAPIevalAggr(Client cntxt, MalBlkPtr mb
 }
 
 str
+PyAPIevalLoader(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+return PyAPIeval(cntxt, mb, stk, pci, 0, 0);
+}
+
+str
 PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
 {
 return PyAPIeval(cntxt, mb, stk, pci, 1, 1);
diff --git a/monetdb5/extras/pyapi/pyapi.h b/monetdb5/extras/pyapi/pyapi.h
--- a/monetdb5/extras/pyapi/pyapi.h
+++ b/monetdb5/extras/pyapi/pyapi.h
@@ -96,6 +96,7 @@ pyapi_export str PyAPIevalStd(Client cnt
 pyapi_export str PyAPIevalAggr(Client cntxt, MalBlkPtr mb, MalStkPtr stk, 
InstrPtr pci);
 pyapi_export str PyAPIevalStdMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, 
InstrPtr pci);
 pyapi_export str PyAPIevalAggrMap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, 
InstrPtr pci);
+pyapi_export str PyAPIevalLoader(Client cntxt, MalBlkPtr mb, MalStkPtr stk, 
InstrPtr pci);
 
 pyapi_export str PyAPIprelude(void *ret);
 
diff --git a/monetdb5/extras/pyapi/pyapi.mal b/monetdb5/extras/pyapi/pyapi.mal
--- a/monetdb5/extras/pyapi/pyapi.mal
+++ b/monetdb5/extras/pyapi/pyapi.mal
@@ -25,6 +25,10 @@ pattern eval_aggr(fptr:ptr,expr:str,arg:
 address PyAPIevalAggr
 comment "grouped aggregates through Python";
 
+pattern eval_loader(fptr:ptr,expr:str,arg:any...):any...
+address PyAPIevalLoader
+comment "loader functions through Python";
+
 # initializer code
 command prelude() :void address PyAPIprelude;
 pyapi.prelude();
@@ -44,6 +48,10 @@ pattern eval_aggr(fptr:ptr,expr:str,arg:
 address PyAPIevalAggr
 comment "grouped aggregates through Python";
 
+pattern eval_loader(fptr:ptr,expr:str,arg:any...):any...
+address PyAPIevalLoader
+comment "loader functions through Python";
+
 module pyapimap;
 
 # The generic Python interface
diff --git a/sql/backends/monet5/sql_gencode.c 
b/sql/backends/monet5/sql_gencode.c
--- a/sql/backends/monet5/sql_gencode.c
+++ b/sql/backends/monet5/sql_gencode.c
@@ -2135,7 +2135,7 @@ static int
if (f->func->lang == FUNC_LANG_R || f->func->lang == 
FUNC_LANG_PY || f->func->lang == FUNC_LANG_MAP_PY)
q = pushStr(mb, q, f->func->query);
/* first dynamic output of copy* functions */
-   if (f->func->type == F_UNION) 
+   if (f->func->type == F_UNION || f->func->type == 
F_LOADER)
q = table_func_create_result(mb, q, f->func, 
f->res);
if (list_length(s->op1->op4.lval))
tpe = tail_type(s->op1->op4.lval->h->data);
@@ -3038,6 +3038,10 @@ backend_create_py_func(backend *be, sql_
f->mod = "pyapi";
f->imp = "eval_aggr";
break;
+   case F_LOADER:
+   f->mod = "pyapi";
+   f->imp = "eval_loader";
+   break;
case  F_PROC: /* no output */
case  F_FUNC:
default: /* ie also F_FILT and F_UNION for now */
diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h
--- a/sql/include/sql_catalog.h
+++ b/sql/include/sql_catalog.h
@@ -281,6 +281,7 @@ typedef struct sql_arg {
 #define F_FILT 4
 #define F_UNION 5
 #define F_ANALYTIC 6
+#define F_LOADER 7
 
 #define IS_FUNC(f) (f->type == F_FUNC)
 #define IS_PROC(f) (f->type == F_PROC)
@@ -288,6 +289,7 @@ typedef struct sql_arg {
 #define IS_FILT(f) (f->type == F_FILT)
 #define IS_UNION(f) (f->type == F_UNION)
 #define IS_ANALYTIC(f) (f->type == F_ANALYTIC)
+#define IS_LOADER(f) (f->type == F_LOADER)
 
 #define FUNC_LANG_INT 0/* internal */
 #define FUNC_LANG_MAL 1 /* create sql external mod.func */
diff --git a/sql/server/rel_psm.c b/sql/server/rel_psm.c
--- a/sql/server/rel_psm.c
+++ b/sql/server/rel_psm.c
@@ -699,10 +699,12 @@ rel_create_func(mvc *sql, dlist *qname, 
char is_table = (res && res->token == SQL_TABLE);
char is_aggr = (type == F_AGGR);
char is_func = (type != F_PROC);
-   char *F = is_aggr?"AGGREGATE":(is_func?"FUNCTION":"PROCEDURE");
+   char is_loader = (type != F_LOADER);
+
+   char *F =

MonetDB: default - fwf fix for windows newlines and general gunk...

2016-06-09 Thread Hannes Muehleisen

Changeset: ee6e38863857 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ee6e38863857
Modified Files:
common/stream/stream.c
Branch: default
Log Message:

fwf fix for windows newlines and general gunk behind fwf area in line


diffs (57 lines):

diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4688,6 +4688,7 @@ typedef struct {
char* out_buf;
size_t out_buf_start;
size_t out_buf_remaining;
+   char* nl_buf;
 } stream_fwf_data;
 
 
@@ -4713,6 +4714,9 @@ stream_fwf_read(stream *s, void *buf, si
}
return buf_written; // skip last line
}
+   // consume to next newline
+   while (fsd->s->read(fsd->s, fsd->nl_buf, 1, 1) == 1 && 
*fsd->nl_buf != '\n');
+
for (field_idx = 0; field_idx < fsd->num_fields; 
field_idx++) {
char *val_start, *val_end;
val_start = fsd->in_buf + in_buf_pos;
@@ -4760,6 +4764,7 @@ stream_fwf_close(stream *s)
free(fsd->widths);
free(fsd->in_buf);
free(fsd->out_buf);
+   free(fsd->nl_buf);
free(fsd);
}
// FIXME destroy(s);
@@ -4778,7 +4783,7 @@ stream_fwf_create (stream *s, size_t num
fsd->num_fields = num_fields;
fsd->widths = widths;
fsd->filler = filler;
-   fsd->line_len = 1; // newline
+   fsd->line_len = 0;
for (i = 0; i < num_fields; i++) {
fsd->line_len += widths[i];
}
@@ -4795,10 +4800,17 @@ stream_fwf_create (stream *s, size_t num
return NULL;
}
fsd->out_buf_remaining = 0;
-
+   fsd->nl_buf = malloc(1);
+   if (!fsd->nl_buf) {
+   free(fsd->in_buf);
+   free(fsd->out_buf);
+   free(fsd);
+   return NULL;
+   }
if ((ns = create_stream(STREAM_FWF_NAME)) == NULL) {
free(fsd->in_buf);
free(fsd->out_buf);
+   free(fsd->nl_buf);
free(fsd);
return NULL;
}
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Exports output

2016-06-08 Thread Hannes Muehleisen

Changeset: b6b5baf0a8c3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b6b5baf0a8c3
Modified Files:
clients/Tests/exports.stable.out
Branch: default
Log Message:

Exports output


diffs (11 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2636,6 +2636,7 @@ stream *socket_rstream(SOCKET socket, co
 stream *socket_wastream(SOCKET socket, const char *name);
 stream *socket_wstream(SOCKET socket, const char *name);
 stream *stream_blackhole_create(void);
+stream *stream_fwf_create(stream *s, size_t num_fields, size_t *widths, char 
filler);
 stream *udp_rastream(const char *hostname, int port, const char *name);
 stream *udp_wastream(const char *hostname, int port, const char *name);
 stream *wbstream(stream *s, size_t buflen);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - COPY INTO from fixed width format files

2016-06-08 Thread Hannes Muehleisen

Changeset: e1a82a5485de for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1a82a5485de
Added Files:
clients/R/Tests/copy_into_fwf.R
clients/R/Tests/copy_into_fwf.reqtests
clients/R/Tests/copy_into_fwf.stable.err
clients/R/Tests/copy_into_fwf.stable.out
Modified Files:
clients/R/Tests/All
clients/R/Tests/dbi.R
clients/R/Tests/deps-test.R
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql.c
sql/common/sql_types.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: default
Log Message:

COPY INTO from fixed width format files


diffs (truncated from 656 to 300 lines):

diff --git a/clients/R/Tests/All b/clients/R/Tests/All
--- a/clients/R/Tests/All
+++ b/clients/R/Tests/All
@@ -4,4 +4,5 @@ HAVE_LIBR?deps-test
 HAVE_LIBR?dbi
 HAVE_LIBR?dplyr
 HAVE_LIBR?dplyr-flights
+HAVE_LIBR?copy_into_fwf
 HAVE_LIBR_WIN32?dbapply
diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -0,0 +1,35 @@
+if (Sys.getenv("TSTTRGDIR") != "") {
+   .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir")))
+}
+library(DBI, quietly = T)
+
+args <- commandArgs(trailingOnly = TRUE)
+dbport <- 5
+dbname <- "mTests_clients_R"
+if (length(args) > 0) 
+   dbport <- args[[1]]
+if (length(args) > 1) 
+   dbname <- args[[2]]
+
+con <- dbConnect(MonetDBLite::MonetDB(), port = dbport, dbname = dbname, wait 
= T)
+stopifnot(dbIsValid(con))
+
+tf <- tempfile()
+
+gdata::write.fwf(mtcars, tf, colnames = FALSE)
+
+if (dbExistsTable(con, "mtcars")) dbRemoveTable(con, "mtcars")
+
+dbBegin(con)
+dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
+
+# delimiters are ineffective for fwf import just set them to make sure they 
dont break stuff
+res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' 
USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)"))
+
+print(dbReadTable(con, "mtcars"))
+
+stopifnot(nrow(dbReadTable(con, "mtcars")) > 1)
+
+dbRollback(con)
+
+print("SUCCESS")
diff --git a/clients/R/Tests/copy_into_fwf.reqtests 
b/clients/R/Tests/copy_into_fwf.reqtests
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.reqtests
@@ -0,0 +1,1 @@
+dbi
diff --git a/clients/R/Tests/copy_into_fwf.stable.err 
b/clients/R/Tests/copy_into_fwf.stable.err
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# builtin opt  gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 30658
+# cmdline opt  mapi_usock = /var/tmp/mtest-16034/.s.monetdb.30658
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_clients_R
+# cmdline opt  embedded_r = yes
+# cmdline opt  gdk_debug = 536870922
+
+# 16:40:23 >  
+# 16:40:23 >  "R" "--vanilla" "--slave" "--args" "30658"
+# 16:40:23 >  
+
+
+# 16:40:23 >  
+# 16:40:23 >  "Done."
+# 16:40:23 >  
+
diff --git a/clients/R/Tests/copy_into_fwf.stable.out 
b/clients/R/Tests/copy_into_fwf.stable.out
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.out
@@ -0,0 +1,71 @@
+stdout of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_clients_R', using 4 threads
+# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit 
integers dynamically linked
+# Found 16.000 GiB available

MonetDB: fixed-width-format - merge with default

2016-06-08 Thread Hannes Muehleisen

Changeset: 07b23de27a1d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=07b23de27a1d
Modified Files:
configure.ag
monetdb5/extras/pyapi/pyapi.c
Branch: fixed-width-format
Log Message:

merge with default


diffs (29 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2359,7 +2359,7 @@ if test "x$enable_pyintegration" != xno;
;;
esac
if test "x$have_pyconfig" = x; then
-   AC_PATH_PROG(PYCMD,python-config,,$XPATH)
+   AC_PATH_PROG(PYCMD,python-config,no,$PATH)
if test "x$PYCMD" = x; then
if test "x$enable_pyintegration" = xyes; then
AC_MSG_ERROR([python-config library required 
for Python integration support])
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -1620,10 +1620,10 @@ str
 MT_lock_set();
 if (!pyapiInitialized) {
 str msg = MAL_SUCCEED;
-char* iar = NULL;
 Py_Initialize();
-PyRun_SimpleString("import numpy");
-import_array1(iar);
+if (PyRun_SimpleString("import numpy") != 0 || _import_array() < 
0) {
+return PyError_CreateException("Failed to initialize embedded 
python", NULL);
+}
 msg = _connection_init();
 marshal_module = PyImport_Import(PyString_FromString("marshal"));
 if (marshal_module == NULL) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - Some cleanup before merge. Niels?

2016-06-08 Thread Hannes Muehleisen

Changeset: 769cc2e46a29 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=769cc2e46a29
Modified Files:
clients/R/Tests/copy_into_fwf.R
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/sql.c
sql/server/rel_updates.c
Branch: fixed-width-format
Log Message:

Some cleanup before merge. Niels?


diffs (90 lines):

diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
--- a/clients/R/Tests/copy_into_fwf.R
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -23,7 +23,8 @@ if (dbExistsTable(con, "mtcars")) dbRemo
 dbBegin(con)
 dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
 
-res <- dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 
4, 5, 6, 6, 2, 2, 2, 2)"))
+# delimiters are ineffective for fwf import just set them to make sure they 
dont break stuff
+res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' 
USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)"))
 
 print(dbReadTable(con, "mtcars"))
 
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4674,11 +4674,7 @@ stream * stream_blackhole_create (void)
 
 
 /* fixed-width format streams */
-
-#define STREAM_FWF_NAME "fwf"
-#define STREAM_FWF_FIELD_SEP '|'
-#define STREAM_FWF_ESCAPE '\\'
-#define STREAM_FWF_RECORD_SEP '\n'
+#define STREAM_FWF_NAME "fwf_ftw"
 
 typedef struct {
stream *s;
diff --git a/common/stream/stream.h b/common/stream/stream.h
--- a/common/stream/stream.h
+++ b/common/stream/stream.h
@@ -87,6 +87,12 @@ typedef __int128_t hge;
 #define ST_READ  0
 #define ST_WRITE 1
 
+/* fwf gets turned into a csv with these parameters */
+#define STREAM_FWF_FIELD_SEP '|'
+#define STREAM_FWF_ESCAPE '\\'
+#define STREAM_FWF_RECORD_SEP '\n'
+#define STREAM_FWF_FILLER ' '
+
 typedef struct stream stream;
 
 /* some os specific initialization */
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3501,17 +3501,26 @@ mvc_import_table_wrap(Client cntxt, MalB
}
widths = malloc(sizeof(size_t) * ncol);
if (!widths) {
-   // TODO: free other stuff
+   mnstr_destroy(ss);
+   GDKfree(tsep);
+   GDKfree(rsep);
+   GDKfree(ssep);
+   GDKfree(ns);
throw(MAL, "sql.copy_from", MAL_MALLOC_FAIL);
}
for (i = 0; i < width_len; i++) {
-   if (fixed_widths[i] == '|') {
+   if (fixed_widths[i] == STREAM_FWF_FIELD_SEP) {
fixed_widths[i] = '\0';
widths[current_width_entry++] = 
(size_t) atoll(val_start);
val_start = fixed_widths + i + 1;
}
}
-   ss = stream_fwf_create(ss, ncol, widths, ' ');
+   /* overwrite other delimiters to the ones the FWF 
stream uses */
+   sprintf((char*) tsep, "%c", STREAM_FWF_FIELD_SEP);
+   sprintf((char*) rsep, "%c", STREAM_FWF_RECORD_SEP);
+   sprintf((char*) ssep, "");
+
+   ss = stream_fwf_create(ss, ncol, widths, 
STREAM_FWF_FILLER);
}
 #if SIZEOF_VOID_P == 4
s = bstream_create(ss, 0x2);
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1167,7 +1167,7 @@ rel_import(mvc *sql, sql_table *t, char 
}
char* fwf_string_cur = fwf_string;
for (dn = fwf_widths->h; dn; dn = dn->next) {
-   fwf_string_cur += sprintf(fwf_string_cur, LLFMT"|", 
dn->data.l_val);
+   fwf_string_cur += sprintf(fwf_string_cur, LLFMT"%c", 
dn->data.l_val, STREAM_FWF_FIELD_SEP);
ncol++;
}
if(list_length(f->res) != ncol) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - PyAPI: Some startup fixes

2016-06-07 Thread Hannes Muehleisen

Changeset: f5e70fd645ad for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f5e70fd645ad
Modified Files:
configure.ag
monetdb5/extras/pyapi/pyapi.c
Branch: default
Log Message:

PyAPI: Some startup fixes


diffs (29 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2359,7 +2359,7 @@ if test "x$enable_pyintegration" != xno;
;;
esac
if test "x$have_pyconfig" = x; then
-   AC_PATH_PROG(PYCMD,python-config,,$XPATH)
+   AC_PATH_PROG(PYCMD,python-config,no,$PATH)
if test "x$PYCMD" = x; then
if test "x$enable_pyintegration" = xyes; then
AC_MSG_ERROR([python-config library required 
for Python integration support])
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -1620,10 +1620,10 @@ str
 MT_lock_set();
 if (!pyapiInitialized) {
 str msg = MAL_SUCCEED;
-char* iar = NULL;
 Py_Initialize();
-PyRun_SimpleString("import numpy");
-import_array1(iar);
+if (PyRun_SimpleString("import numpy") != 0 || _import_array() < 
0) {
+return PyError_CreateException("Failed to initialize embedded 
python", NULL);
+}
 msg = _connection_init();
 marshal_module = PyImport_Import(PyString_FromString("marshal"));
 if (marshal_module == NULL) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - fix and test case

2016-06-07 Thread Hannes Muehleisen

Changeset: 3e166b600cad for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3e166b600cad
Added Files:
clients/R/Tests/copy_into_fwf.stable.err
clients/R/Tests/copy_into_fwf.stable.out
Modified Files:
clients/R/Tests/copy_into_fwf.R
clients/R/Tests/dbi.R
sql/backends/monet5/sql.c
Branch: fixed-width-format
Log Message:

fix and test case


diffs (157 lines):

diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
--- a/clients/R/Tests/copy_into_fwf.R
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -23,11 +23,11 @@ if (dbExistsTable(con, "mtcars")) dbRemo
 dbBegin(con)
 dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
 
-dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 
6, 6, 2, 2, 2, 2)"))
+res <- dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 
4, 5, 6, 6, 2, 2, 2, 2)"))
 
 print(dbReadTable(con, "mtcars"))
 
-stopifnot(nrow(dbReadTable(con, "mtcars")) < 1)
+stopifnot(nrow(dbReadTable(con, "mtcars")) > 1)
 
 dbRollback(con)
 
diff --git a/clients/R/Tests/copy_into_fwf.stable.err 
b/clients/R/Tests/copy_into_fwf.stable.err
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# builtin opt  gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 30658
+# cmdline opt  mapi_usock = /var/tmp/mtest-16034/.s.monetdb.30658
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_clients_R
+# cmdline opt  embedded_r = yes
+# cmdline opt  gdk_debug = 536870922
+
+# 16:40:23 >  
+# 16:40:23 >  "R" "--vanilla" "--slave" "--args" "30658"
+# 16:40:23 >  
+
+
+# 16:40:23 >  
+# 16:40:23 >  "Done."
+# 16:40:23 >  
+
diff --git a/clients/R/Tests/copy_into_fwf.stable.out 
b/clients/R/Tests/copy_into_fwf.stable.out
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.out
@@ -0,0 +1,71 @@
+stdout of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_clients_R', using 4 threads
+# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit 
integers dynamically linked
+# Found 16.000 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2016 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:30658/
+# Listening for UNIX domain connection requests on 
mapi:monetdb:///var/tmp/mtest-16034/.s.monetdb.30658
+# MonetDB/SQL module loaded
+# MonetDB/R   module loaded
+
+Ready.
+
+# 16:40:23 >  
+# 16:40:23 >  "R" "--vanilla" "--slave" "--args" "30658"
+# 16:40:23 >  
+
+[1] TRUE
+
+  SQL  CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp 
DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE 
PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, 
gear DOUBLE PRECISION, carb DOUBLE PRECISION)
+  ROWS Fetched: 0 [complete]
+   Changed: NA
+mpg cyl  disp  hp dratwt  qsec vs am gear carb
+1  21.0   6 160.0 110 3.90 2.620 16.46  0  144
+2  21.0   6 160.0 110 3.90 2.875 17.02  0  144
+3  22.8   4 108.0  93 3.85 2.320 18.61  1  141
+4  21.4   6 258.0 110 3.08 3.215 19.44  1  031
+5  18.7   8 360.0 175 3.15 3.440 17.02  0  032
+6  18.1   6 225.0 105 2.76 3.460 20.22  1  031
+7  14.3   8 360.0 245 3.21 3.570 15.84  0  034
+8  24.4   4 146.7  62 3.69 3.190 20.00  1  042
+9  22.8   4

MonetDB: fixed-width-format - Test case

2016-06-07 Thread Hannes Muehleisen

Changeset: 24ecd74d3d62 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=24ecd74d3d62
Added Files:
clients/R/Tests/copy_into_fwf.R
clients/R/Tests/copy_into_fwf.reqtests
Modified Files:
clients/R/Tests/All
clients/R/Tests/deps-test.R
common/stream/stream.c
Branch: fixed-width-format
Log Message:

Test case


diffs (76 lines):

diff --git a/clients/R/Tests/All b/clients/R/Tests/All
--- a/clients/R/Tests/All
+++ b/clients/R/Tests/All
@@ -4,4 +4,5 @@ HAVE_LIBR?deps-test
 HAVE_LIBR?dbi
 HAVE_LIBR?dplyr
 HAVE_LIBR?dplyr-flights
+HAVE_LIBR?copy_into_fwf
 HAVE_LIBR_WIN32?dbapply
diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -0,0 +1,34 @@
+if (Sys.getenv("TSTTRGDIR") != "") {
+   .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir")))
+}
+library(DBI, quietly = T)
+
+args <- commandArgs(trailingOnly = TRUE)
+dbport <- 5
+dbname <- "mTests_clients_R"
+if (length(args) > 0) 
+   dbport <- args[[1]]
+if (length(args) > 1) 
+   dbname <- args[[2]]
+
+con <- dbConnect(MonetDBLite::MonetDB(), port = dbport, dbname = dbname, wait 
= T)
+stopifnot(dbIsValid(con))
+
+tf <- tempfile()
+
+gdata::write.fwf(mtcars, tf, colnames = FALSE)
+
+if (dbExistsTable(con, "mtcars")) dbRemoveTable(con, "mtcars")
+
+dbBegin(con)
+dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
+
+dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 4, 5, 
6, 6, 2, 2, 2, 2)"))
+
+print(dbReadTable(con, "mtcars"))
+
+stopifnot(nrow(dbReadTable(con, "mtcars")) < 1)
+
+dbRollback(con)
+
+print("SUCCESS")
diff --git a/clients/R/Tests/copy_into_fwf.reqtests 
b/clients/R/Tests/copy_into_fwf.reqtests
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.reqtests
@@ -0,0 +1,1 @@
+dbi
diff --git a/clients/R/Tests/deps-test.R b/clients/R/Tests/deps-test.R
--- a/clients/R/Tests/deps-test.R
+++ b/clients/R/Tests/deps-test.R
@@ -4,5 +4,5 @@ dd <- capture.output(suppressMessages(su
repos <- 'http://cran.rstudio.com/'
if(length(np)) install.packages(np, repos=repos, quiet=T)
update.packages(repos=repos, ask=F, oldPkgs=lp, quiet=T)
-   })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13'))
+   })(c('Rcpp', 'dplyr', 'Lahman', 'nycflights13', 'gdata'))
 })))
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4791,7 +4791,7 @@ stream_fwf_create (stream *s, size_t num
free(fsd);
return NULL;
}
-   out_buf_len = fsd->line_len * 2; // TODO: what if this is not enough?
+   out_buf_len = fsd->line_len * 3;
fsd->out_buf = malloc(out_buf_len);
if (!fsd->out_buf) {
free(fsd->in_buf);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - merge with default

2016-06-07 Thread Hannes Muehleisen

Changeset: 4defb6578c55 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4defb6578c55
Added Files:
debian/libmonetdb-client8.install
debian/libmonetdb-stream8.install
debian/libmonetdb13.install
sql/backends/monet5/Tests/pyapi29.sql
sql/backends/monet5/Tests/pyapi29.stable.err
sql/backends/monet5/Tests/pyapi29.stable.out
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.oid32
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.oid32
Removed Files:
buildtools/ChangeLog.Jul2015
clients/ChangeLog.Jul2015
clients/mapilib/ChangeLog.Jul2015
clients/python2/ChangeLog.Jul2015
clients/python3/ChangeLog.Jul2015
common/stream/ChangeLog.Jul2015
debian/libmonetdb-client7.install
debian/libmonetdb-stream7.install
debian/libmonetdb12.install
gdk/ChangeLog.Jul2015
geom/ChangeLog.Jul2015
java/ChangeLog.Jul2015
monetdb5/ChangeLog.Jul2015
sql/ChangeLog.Jul2015
testing/ChangeLog.Jul2015
tools/merovingian/ChangeLog.Jul2015
Modified Files:
.hgtags
MonetDB.spec
NT/installer32/MonetDB-ODBC-Installer.vdproj
NT/installer32/MonetDB5-Geom-Module.vdproj
NT/installer32/MonetDB5-SQL-Installer.vdproj
NT/installer64/MonetDB-ODBC-Installer.vdproj
NT/installer64/MonetDB5-Geom-Module.vdproj
NT/installer64/MonetDB5-SQL-Installer.vdproj
NT/monetdb_config.h.in
NT/rules.msc
buildtools/ChangeLog-Archive
buildtools/ChangeLog.Jun2016
buildtools/doc/windowsbuild.rst
clients/ChangeLog-Archive
clients/ChangeLog.Jun2016
clients/R/Tests/dbi.R
clients/R/Tests/dbi.stable.out
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
clients/python2/setup.py
clients/python3/setup.py
common/utils/msabaoth.c
configure.ag
debian/changelog
debian/control
debian/fix-deb.sh
gdk/ChangeLog-Archive
gdk/ChangeLog.Jun2016
gdk/gdk_bbp.c
gdk/gdk_logger.c
gdk/gdk_utils.c
gdk/libbat.rc
geom/ChangeLog-Archive
geom/ChangeLog.Jun2016
java/ChangeLog-Archive
java/ChangeLog.Jun2016
java/Makefile.ag
java/build.properties
java/pom.xml
java/release.txt
java/src/main/java/nl/cwi/monetdb/jdbc/MonetDatabaseMetaData.java
java/src/main/java/nl/cwi/monetdb/jdbc/MonetPreparedStatement.java
libversions
monetdb5/ChangeLog-Archive
monetdb5/ChangeLog.Jun2016
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
monetdb5/extras/rapi/rapi.R
monetdb5/extras/rapi/rapi.c
monetdb5/mal/Tests/tst010.stable.err
monetdb5/mal/mal.c
monetdb5/mal/mal.h
monetdb5/mal/mal_dataflow.c
monetdb5/mal/mal_debugger.c
monetdb5/mal/mal_debugger.h
monetdb5/mal/mal_function.c
monetdb5/mal/mal_function.h
monetdb5/mal/mal_instruction.c
monetdb5/mal/mal_instruction.h
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_profiler.c
monetdb5/modules/atoms/json.c
monetdb5/modules/mal/groupby.c
monetdb5/modules/mal/mdb.c
monetdb5/modules/mal/mdb.h
monetdb5/modules/mal/mdb.mal
monetdb5/modules/mal/tablet.c
monetdb5/optimizer/Tests/dataflow3.stable.out
monetdb5/optimizer/Tests/inline02.stable.out
monetdb5/optimizer/Tests/inline03.stable.out
monetdb5/optimizer/Tests/inline04.stable.out
monetdb5/optimizer/Tests/inline06.stable.out
monetdb5/optimizer/opt_aliases.c
monetdb5/optimizer/opt_candidates.c
monetdb5/optimizer/opt_coercion.c
monetdb5/optimizer/opt_commonTerms.c
monetdb5/optimizer/opt_constants.c
monetdb5/optimizer/opt_costModel.c
monetdb5/optimizer/opt_dataflow.c
monetdb5/optimizer/opt_deadcode.c
monetdb5/optimizer/opt_evaluate.c
monetdb5/optimizer/opt_factorize.c
monetdb5/optimizer/opt_garbageCollector.c
monetdb5/optimizer/opt_generator.c
monetdb5/optimizer/opt_inline.c
monetdb5/optimizer/opt_json.c
monetdb5/optimizer/opt_matpack.c
monetdb5/optimizer/opt_mergetable.c
monetdb5/optimizer/opt_mitosis.c
monetdb5/optimizer/opt_multiplex.c
monetdb5/optimizer/opt_pipes.c
monetdb5/optimizer/opt_profiler.c
monetdb5/optimizer/opt_projectionpath.c
monetdb5/optimizer/opt_pushselect.c
monetdb5/optimizer/opt_querylog.c
monetdb5/optimizer/opt_recycler.c

MonetDB: fixed-width-format - Remove first stab

2016-06-07 Thread Hannes Muehleisen

Changeset: b88502716f21 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b88502716f21
Removed Files:
sql/backends/monet5/Tests/fwf.malC
sql/backends/monet5/fwf.c
sql/backends/monet5/fwf.h
sql/backends/monet5/fwf.mal
Modified Files:
sql/backends/monet5/Makefile.ag
Branch: fixed-width-format
Log Message:

Remove first stab


diffs (248 lines):

diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag
--- a/sql/backends/monet5/Makefile.ag
+++ b/sql/backends/monet5/Makefile.ag
@@ -46,8 +46,7 @@ lib__sql = {
sql_round.c sql_round_impl.h sql_bat2time.c \
sql_fround.c sql_fround_impl.h \
sql_orderidx.c sql_orderidx.h \
-   sql_rank.c sql_rank.h \
-   fwf.c fwf.h
+   sql_rank.c sql_rank.h
LIBS = ../../server/libsqlserver \
   ../../storage/libstore \
   ../../storage/bat/libbatstore \
@@ -65,8 +64,7 @@ headers_mal = {
DIR = libdir/monetdb5
SOURCES = sql_aggr_bte.mal  sql_aggr_flt.mal sql_aggr_dbl.mal  
sql_aggr_int.mal  sql_aggr_lng.mal \
sql_aggr_sht.mal  sql_decimal.mal  sql_inspect.mal \
-   sql_rank.mal sql.mal \
-   fwf.mal
+   sql_rank.mal sql.mal
 }
 
 headers_mal_hge = {
diff --git a/sql/backends/monet5/Tests/fwf.malC 
b/sql/backends/monet5/Tests/fwf.malC
deleted file mode 100644
--- a/sql/backends/monet5/Tests/fwf.malC
+++ /dev/null
@@ -1,25 +0,0 @@
-# write.fwf(mtcars, "/tmp/mtcars", header=F)
-
-# CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE 
PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION, 
qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE 
PRECISION, carb DOUBLE PRECISION)
-
-#4   2 6 4   56 6 2 2 2 2
-# 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4
-
-sql.start2();
-
-bint:= bat.new(:oid,:int);
-bat.append(bint,4:int);
-bat.append(bint,2:int);
-bat.append(bint,6:int);
-bat.append(bint,4:int);
-bat.append(bint,5:int);
-bat.append(bint,6:int);
-bat.append(bint,6:int);
-bat.append(bint,2:int);
-bat.append(bint,2:int);
-bat.append(bint,2:int);
-bat.append(bint,2:int);
-
-include fwf;
-
-fwf.load("sys", "mtcars", "/tmp/mtcars", bint, 32:bte);
diff --git a/sql/backends/monet5/fwf.c b/sql/backends/monet5/fwf.c
deleted file mode 100644
--- a/sql/backends/monet5/fwf.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/* Ghetto programming warning. */
-
-#include "monetdb_config.h"
-#include "mal.h"
-#include "mal_stack.h"
-#include "mal_linker.h"
-#include "gdk_utils.h"
-#include "gdk.h"
-#include "mmath.h"
-#include "sql_catalog.h"
-#include "sql_execute.h"
-#include "fwf.h"
-
-str fwf_load(mvc *m, char* schema, char* table, char* filename, BAT *widths, 
char padding) {
-   FILE *f;
-   size_t fsize;
-struct stat stb;
-char* fptr, *fcur = NULL;
-size_t i;
-size_t width_sum = 0;
-   sql_schema *s;
-   sql_table *t;
-   node *n;
-   str msg = MAL_SUCCEED;
-   size_t ncol;
-   size_t approx_nrows;
-   size_t row = 0;
-   BAT** appends;
-   char* line = NULL;
-   int *widths_ptr = (int*) widths->T->heap.base;
-
-   s = mvc_bind_schema(m, schema);
-   if (s == NULL)
-   throw(SQL, "sql.append", "Schema missing");
-   t = mvc_bind_table(m, s, table);
-   if (t == NULL)
-   throw(SQL, "sql.append", "Table missing");
-
-   f = fopen(filename, "r");
-   if (fstat(fileno(f), ) != 0) {
-   msg = createException(MAL, "fwf.load", "Could not stat file");
-   goto cleanup;
-}
-fsize = (size_t) stb.st_size;
-fptr = GDKmmap(filename, MMAP_READ, fsize);
-   if (!fptr) {
-   msg = createException(MAL, "fwf.load", "Could not map file");
-   goto cleanup;
-   }
-
-   ncol = t->columns.set->cnt;
-   if (BATcount(widths) != ncol) {
-   msg = createException(MAL, "fwf.load", "Incorrect number of 
widths supplied");
-   goto cleanup;
-   }
-
-   appends = malloc(sizeof(BAT*)*ncol);
-   if (!appends) {
-   msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
-   goto cleanup;
-   }
-
-   for (i = 0; i < ncol; i++) {
-   width_sum += widths_ptr[i];
-   }
-   line = GDKmalloc(width_sum + ncol + 1);
-   if (!line) {
-   msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
-   goto cleanup;
-   }
-
-   approx_nrows = fsize/width_sum;
-   i = 0;
-   for (n = t->columns.set->h; n; n = n->next) {
-   sql_column *c = n->data;
-   appends[i] = BATnew(TYPE_void, c->type.type->localtype, 
approx_nrows, TRANSIENT) ;
-   if (!appends[i]) {
-   msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
-

MonetDB: fixed-width-format - COPY INTO table FWF (1,2,3); now w...

2016-06-07 Thread Hannes Muehleisen

Changeset: eb9a9582ba77 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=eb9a9582ba77
Modified Files:
sql/backends/monet5/sql.c
sql/common/sql_types.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: fixed-width-format
Log Message:

COPY INTO table FWF (1,2,3); now works


diffs (267 lines):

diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3410,10 +3410,16 @@ mvc_import_table_wrap(Client cntxt, MalB
lng *offset = getArgReference_lng(stk, pci, pci->retc + 7);
int *locked = getArgReference_int(stk, pci, pci->retc + 8);
int *besteffort = getArgReference_int(stk, pci, pci->retc + 9);
+   char *fixed_widths = NULL;
str msg = MAL_SUCCEED;
bstream *s = NULL;
stream *ss;
 
+   if (pci->argc - pci->retc > 10) {
+   fixed_widths = *getArgReference_str(stk, pci, pci->retc + 10);
+
+   }
+
(void) mb;  /* NOT USED */
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
@@ -3483,14 +3489,29 @@ mvc_import_table_wrap(Client cntxt, MalB
return msg;
}
GDKfree(fn);
-
-   // FIXME
-   if (GDKgetenv_isyes("testfwf")) {
-   fprintf(stderr, "### FWF IMPORT ON %s ###\n", *fname);
-   size_t *widths = malloc(sizeof(size_t)*400);
-   size_t i = 0;
-   widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 9;  
widths[i++] = 12;  widths[i++] = 8;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 8;  
widths[i++] = 8;  widths[i++] = 30;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 9;  widths[i++] = 4;  widths[i++] = 4;  widths[i++] = 2;  
widths[i++] = 8;  widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 9;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 4;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  wi
 dths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 3;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 4;  
widths[i++] = 2;  widths[i++] = 8;  widths[i++] = 2;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i+
 +] = 2;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 
3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 9;  widths[i++] = 9;  
widths[i++] = 9;  widths[i++] = 8;  widths[i++] = 8;  widths[i++] = 8;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 9;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 9;  widths[i++] = 9;  widths[i++] = 9;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 2;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;
   widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 9;  widths[i++] = 9;  
widths[i++] = 9;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 2;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  
widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;  widths[i++] = 3;

1 2 3 4 5 6 7 8 9 10 >

1 - 100 of 905 matches

Mail list logo