date:20160608

MonetDB: default - Reduce optimizer cost

2016-06-08 Thread Martin Kersten

Changeset: 727821377f76 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=727821377f76
Modified Files:
monetdb5/optimizer/opt_generator.c
monetdb5/optimizer/opt_matpack.c
Branch: default
Log Message:

Reduce optimizer cost


diffs (51 lines):

diff --git a/monetdb5/optimizer/opt_generator.c 
b/monetdb5/optimizer/opt_generator.c
--- a/monetdb5/optimizer/opt_generator.c
+++ b/monetdb5/optimizer/opt_generator.c
@@ -159,11 +159,10 @@ OPTgeneratorImplementation(Client cntxt,
 #endif
 
 /* Defense line against incorrect plans */
-if( actions > 0){
-chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
-chkFlow(cntxt->fdout, mb);
-chkDeclarations(cntxt->fdout, mb);
-}
+   /* all new/modified statements are already checked */
+   //chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
+   //chkFlow(cntxt->fdout, mb);
+   //chkDeclarations(cntxt->fdout, mb);
 /* keep all actions taken as a post block comment */
 snprintf(buf,256,"%-20s actions=%2d time=" LLFMT " 
usec","generator",actions,GDKusec() - usec);
 newComment(mb,buf);
diff --git a/monetdb5/optimizer/opt_matpack.c b/monetdb5/optimizer/opt_matpack.c
--- a/monetdb5/optimizer/opt_matpack.c
+++ b/monetdb5/optimizer/opt_matpack.c
@@ -40,6 +40,7 @@ OPTmatpackImplementation(Client cntxt, M
setVarType(mb,v,getArgType(mb,p,1));
q = pushArgument(mb, q, getArg(p,1));
q = pushInt(mb,q, p->argc - p->retc);
+   typeChecker(cntxt->fdout, cntxt->nspace,mb,q,TRUE);
 
for ( j = 2; j < p->argc; j++) {
q = newStmt(mb,matRef, packIncrementRef);
@@ -47,6 +48,7 @@ OPTmatpackImplementation(Client cntxt, M
q = pushArgument(mb, q, getArg(p,j));
setVarType(mb,getArg(q,0),getVarType(mb,v));
v = getArg(q,0);
+   typeChecker(cntxt->fdout, 
cntxt->nspace,mb,q,TRUE);
}
getArg(q,0) = getArg(p,0);
freeInstruction(p);
@@ -62,9 +64,9 @@ OPTmatpackImplementation(Client cntxt, M
 
 /* Defense line against incorrect plans */
 if( actions > 0){
-chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
-chkFlow(cntxt->fdout, mb);
-chkDeclarations(cntxt->fdout, mb);
+//chkTypes(cntxt->fdout, cntxt->nspace, mb, FALSE);
+//chkFlow(cntxt->fdout, mb);
+//chkDeclarations(cntxt->fdout, mb);
 }
 /* keep all actions taken as a post block comment */
 snprintf(buf,256,"%-20s actions=%2d time=" LLFMT " 
usec","matpack",actions,GDKusec() - usec);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: iot - Finished stream creation through webserver with M...

2016-06-08 Thread Pedro Ferreira

Changeset: 4b3ab86b53d0 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4b3ab86b53d0
Modified Files:
clients/iotapi/src/main.py
clients/iotclient/documentation/iot_server_arguments.rst
clients/iotclient/documentation/restful_resources.rst
clients/iotclient/src/Flask/restresources.py
clients/iotclient/src/Settings/mapiconnection.py
clients/iotclient/src/Streams/datatypes.py
clients/iotclient/src/Streams/jsonschemas.py
clients/iotclient/src/Streams/streampolling.py
clients/iotclient/src/Streams/streams.py
clients/iotclient/src/Streams/streamscontext.py
clients/iotclient/src/Streams/streamscreator.py
clients/iotclient/src/main.py
clients/iotclient/tests/datatypesinsertstests.py
Branch: iot
Log Message:

Finished stream creation through webserver with MonetDB, now only the stream 
polling is remaining


diffs (truncated from 954 to 300 lines):

diff --git a/clients/iotapi/src/main.py b/clients/iotapi/src/main.py
--- a/clients/iotapi/src/main.py
+++ b/clients/iotapi/src/main.py
@@ -65,7 +65,7 @@ def main():
 help='Baskets location directory (default: %s)' % 
DEFAULT_FILESYSTEM, metavar='DIRECTORY')
 parser.add_argument('-l', '--log', type=check_path, nargs='?', 
default=DEFAULT_LOGGING,
 help='Logging file location (default: %s)' % 
DEFAULT_LOGGING, metavar='FILE_PATH')
-parser.add_argument('-po', '--polling', type=check_positive_int, 
nargs='?', default=60,
+parser.add_argument('-po', '--polling', type=check_positive_int, 
nargs='?', default=60, metavar='POLLING',
 help='Polling interval in seconds to the database for 
streams updates (default: 60)')
 parser.add_argument('-sh', '--shost', type=check_ipv4_address, nargs='?', 
default='0.0.0.0',
 help='Web API server host (default: 0.0.0.0)', 
metavar='HOST')
diff --git a/clients/iotclient/documentation/iot_server_arguments.rst 
b/clients/iotclient/documentation/iot_server_arguments.rst
--- a/clients/iotclient/documentation/iot_server_arguments.rst
+++ b/clients/iotclient/documentation/iot_server_arguments.rst
@@ -32,19 +32,19 @@ Set the filesystem directory where the b
 
 Location of logfile. On the logfile is reported when streams are created or 
removed, when tuples are inserted and when the baskets are flushed. By default 
in UNIX systems is :code:`/var/log/iot/iotserver.log`, while on Windows is the 
:code:`iotserver.log` on the directory where the :code:`main.py` script was 
called.
 
-Host Identifier

+Web Server Behavior
+---
 
-If the *identifier* parameter is provided, an extra column on streams will be 
added with a custom name of the host for later identification.
-
-**-i  - -identifier**
-
-Use a host identifier for every new stream.
+If a stream is created with a hostname, an extra column on streams will be 
added with a custom name of the host for later identification.
 
 **-n  - -name**
 
 Host identifier name. By default is the host's MAC address.
 
+**-po  - -polling**
+
+Set the polling interval in seconds to MonetDB database for streams updates. 
By default is :code:`60` seconds.
+
 Web Server Listening
 
 
@@ -69,7 +69,7 @@ Listening port of the administration ser
 Database Connection
 ---
 
-Credentials for the MAPI connection to MonetDB database.
+Credentials for the MAPI connection to the MonetDB database.
 
 .. note:: The user's password will be prompted during the initialization of 
the server.
 
diff --git a/clients/iotclient/documentation/restful_resources.rst 
b/clients/iotclient/documentation/restful_resources.rst
--- a/clients/iotclient/documentation/restful_resources.rst
+++ b/clients/iotclient/documentation/restful_resources.rst
@@ -20,13 +20,14 @@ The administration server provides resou
 
 **GET**
 
-Returns a JSON file with details about all the streams currently created on 
the webserver. For each stream besides its schema and name, it provides the 
currently number of tuples inserted on the baskets per column, description of 
columns (`See data types `__), the flushing 
method (`See streams creation for details 
`__). An example is shown bellow:
+Returns a JSON file with details about all the streams currently created on 
the webserver. For each stream besides its schema, name and if has the hostname 
column, it provides the currently number of tuples inserted on the baskets per 
column, description of columns (`See data types 
`__), the flushing method (`See streams 
creation for details `__). An example 
is shown bellow:
 
 .. code-block:: json
 
[
  {
"tuples_inserted_per_basket": 1,
+   "hostname": false,
"columns": [

MonetDB: Jun2016 - Typo.

2016-06-08 Thread Sjoerd Mullender

Changeset: 25a2dcd038b8 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=25a2dcd038b8
Modified Files:
MonetDB.spec
Branch: Jun2016
Log Message:

Typo.


diffs (12 lines):

diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -9,7 +9,7 @@
 %define bits 32
 %else
 %define bits 64
-%define with_int128=1
+%define with_int128 1
 %endif
 
 # only add .oidXX suffix if oid size differs from bit size
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - approved output after fwf merge

2016-06-08 Thread Niels Nes

Changeset: 8b00131c3de5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8b00131c3de5
Modified Files:
clients/Tests/SQL-dump.stable.out
clients/Tests/SQL-dump.stable.out.int128
monetdb5/extras/mal_optimizer_template/Tests/opt_sql_append.stable.out
sql/common/sql_list.c
sql/include/sql_list.h
sql/server/rel_optimizer.c
sql/test/BugTracker-2009/Tests/copy_multiple_files.SF-2902320.stable.out

sql/test/BugTracker-2009/Tests/segfault_when_quitting_debugger.SF-2538837.stable.out
sql/test/Tests/systemfunctions.stable.out
sql/test/Tests/systemfunctions.stable.out.int128
sql/test/leaks/Tests/check1.stable.out.int128
sql/test/leaks/Tests/check2.stable.out.int128
sql/test/leaks/Tests/check3.stable.out.int128
sql/test/leaks/Tests/check4.stable.out.int128
sql/test/leaks/Tests/check5.stable.out.int128
Branch: default
Log Message:

approved output after fwf merge
optimized join reorder more.


diffs (truncated from 485 to 300 lines):

diff --git a/clients/Tests/SQL-dump.stable.out 
b/clients/Tests/SQL-dump.stable.out
--- a/clients/Tests/SQL-dump.stable.out
+++ b/clients/Tests/SQL-dump.stable.out
@@ -3438,7 +3438,7 @@ drop function pcre_replace(string, strin
 [ "sys",   "concat",   "arg_1","varchar",  0,  0,  
1,  "arg_2","varchar",  0,  0,  1,  NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL]
 [ "sys",   "contains", "a","geometry", 0,  0,  1,  
"x","double",   53, 0,  1,  "y","double",   53, 
0,  1,  NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL]
 [ "sys",   "copyfrom", "arg_1","varchar",  0,  0,  
1,  "arg_2","varchar",  0,  0,  1,  NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL]
-[ "sys",   "copyfrom", "arg_1","varchar",  0,  0,  
1,  "arg_2","varchar",  0,  0,  1,  "arg_3",
"varchar",  0,  0,  1,  "arg_4","varchar",  0,  
0,  1,  "arg_5","varchar",  0,  0,  1,  
"arg_6","varchar",  0,  0,  1,  "arg_7",
"bigint",   64, 0,  1,  "arg_8","bigint",   64, 
0,  1,  "arg_9","int",  32, 0,  1,  NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL]
+[ "sys",   "copyfrom", "arg_1","varchar",  0,  0,  
1,  "arg_2","varchar",  0,  0,  1,  "arg_3",
"varchar",  0,  0,  1,  "arg_4","varchar",  0,  
0,  1,  "arg_5","varchar",  0,  0,  1,  
"arg_6","varchar",  0,  0,  1,  "arg_7",
"bigint",   64, 0,  1,  "arg_8","bigint",   64, 
0,  1,  "arg_9","int",  32, 0,  1,  "arg_:",
"varchar",  0,  0,  1,  NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL
]
 [ "sys",   "corr", "e1",   "bigint",   64, 0,  1,  "e2",   
"bigint",   64,

MonetDB: default - Exports output

2016-06-08 Thread Hannes Muehleisen

Changeset: b6b5baf0a8c3 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b6b5baf0a8c3
Modified Files:
clients/Tests/exports.stable.out
Branch: default
Log Message:

Exports output


diffs (11 lines):

diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2636,6 +2636,7 @@ stream *socket_rstream(SOCKET socket, co
 stream *socket_wastream(SOCKET socket, const char *name);
 stream *socket_wstream(SOCKET socket, const char *name);
 stream *stream_blackhole_create(void);
+stream *stream_fwf_create(stream *s, size_t num_fields, size_t *widths, char 
filler);
 stream *udp_rastream(const char *hostname, int port, const char *name);
 stream *udp_wastream(const char *hostname, int port, const char *name);
 stream *wbstream(stream *s, size_t buflen);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - COPY INTO from fixed width format files

2016-06-08 Thread Hannes Muehleisen

Changeset: e1a82a5485de for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e1a82a5485de
Added Files:
clients/R/Tests/copy_into_fwf.R
clients/R/Tests/copy_into_fwf.reqtests
clients/R/Tests/copy_into_fwf.stable.err
clients/R/Tests/copy_into_fwf.stable.out
Modified Files:
clients/R/Tests/All
clients/R/Tests/dbi.R
clients/R/Tests/deps-test.R
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql.c
sql/common/sql_types.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: default
Log Message:

COPY INTO from fixed width format files


diffs (truncated from 656 to 300 lines):

diff --git a/clients/R/Tests/All b/clients/R/Tests/All
--- a/clients/R/Tests/All
+++ b/clients/R/Tests/All
@@ -4,4 +4,5 @@ HAVE_LIBR?deps-test
 HAVE_LIBR?dbi
 HAVE_LIBR?dplyr
 HAVE_LIBR?dplyr-flights
+HAVE_LIBR?copy_into_fwf
 HAVE_LIBR_WIN32?dbapply
diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -0,0 +1,35 @@
+if (Sys.getenv("TSTTRGDIR") != "") {
+   .libPaths(c(.libPaths(), paste0(Sys.getenv("TSTTRGDIR"),"/rlibdir")))
+}
+library(DBI, quietly = T)
+
+args <- commandArgs(trailingOnly = TRUE)
+dbport <- 5
+dbname <- "mTests_clients_R"
+if (length(args) > 0) 
+   dbport <- args[[1]]
+if (length(args) > 1) 
+   dbname <- args[[2]]
+
+con <- dbConnect(MonetDBLite::MonetDB(), port = dbport, dbname = dbname, wait 
= T)
+stopifnot(dbIsValid(con))
+
+tf <- tempfile()
+
+gdata::write.fwf(mtcars, tf, colnames = FALSE)
+
+if (dbExistsTable(con, "mtcars")) dbRemoveTable(con, "mtcars")
+
+dbBegin(con)
+dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
+
+# delimiters are ineffective for fwf import just set them to make sure they 
dont break stuff
+res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' 
USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)"))
+
+print(dbReadTable(con, "mtcars"))
+
+stopifnot(nrow(dbReadTable(con, "mtcars")) > 1)
+
+dbRollback(con)
+
+print("SUCCESS")
diff --git a/clients/R/Tests/copy_into_fwf.reqtests 
b/clients/R/Tests/copy_into_fwf.reqtests
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.reqtests
@@ -0,0 +1,1 @@
+dbi
diff --git a/clients/R/Tests/copy_into_fwf.stable.err 
b/clients/R/Tests/copy_into_fwf.stable.err
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.err
@@ -0,0 +1,35 @@
+stderr of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# builtin opt  gdk_dbpath = /tmp/fuckit/var/monetdb5/dbfarm/demo
+# builtin opt  gdk_debug = 0
+# builtin opt  gdk_vmtrim = no
+# builtin opt  monet_prompt = >
+# builtin opt  monet_daemon = no
+# builtin opt  mapi_port = 5
+# builtin opt  mapi_open = false
+# builtin opt  mapi_autosense = false
+# builtin opt  sql_optimizer = default_pipe
+# builtin opt  sql_debug = 0
+# cmdline opt  gdk_nr_threads = 0
+# cmdline opt  mapi_open = true
+# cmdline opt  mapi_port = 30658
+# cmdline opt  mapi_usock = /var/tmp/mtest-16034/.s.monetdb.30658
+# cmdline opt  monet_prompt = 
+# cmdline opt  gdk_dbpath = /tmp/fuckit/var/MonetDB/mTests_clients_R
+# cmdline opt  embedded_r = yes
+# cmdline opt  gdk_debug = 536870922
+
+# 16:40:23 >  
+# 16:40:23 >  "R" "--vanilla" "--slave" "--args" "30658"
+# 16:40:23 >  
+
+
+# 16:40:23 >  
+# 16:40:23 >  "Done."
+# 16:40:23 >  
+
diff --git a/clients/R/Tests/copy_into_fwf.stable.out 
b/clients/R/Tests/copy_into_fwf.stable.out
new file mode 100644
--- /dev/null
+++ b/clients/R/Tests/copy_into_fwf.stable.out
@@ -0,0 +1,71 @@
+stdout of test 'copy_into_fwf` in directory 'clients/R` itself:
+
+
+# 16:40:22 >  
+# 16:40:22 >  "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" 
"mapi_open=true" "--set" "mapi_port=30658" "--set" 
"mapi_usock=/var/tmp/mtest-16034/.s.monetdb.30658" "--set" "monet_prompt=" 
"--forcemito" "--dbpath=/tmp/fuckit/var/MonetDB/mTests_clients_R" "--set" 
"embedded_r=yes"
+# 16:40:22 >  
+
+# MonetDB 5 server v11.24.0
+# This is an unreleased version
+# Serving database 'mTests_clients_R', using 4 threads
+# Compiled for x86_64-apple-darwin15.5.0/64bit with 64bit OIDs and 128bit 
integers dynamically linked
+# Found 16.000 GiB available

MonetDB: fixed-width-format - fwf is not supported on stdin (for...

2016-06-08 Thread Niels Nes

Changeset: 1da670310b51 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=1da670310b51
Modified Files:
sql/server/sql_parser.y
Branch: fixed-width-format
Log Message:

fwf is not supported on stdin (for now)..


diffs (21 lines):

diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -2497,7 +2497,7 @@ copyfrom_stmt:
  append_int(l, $13);
  append_list(l, $14);
  $$ = _symbol_create_list( SQL_COPYFROM, l ); }
-  | COPY opt_nr INTO qname opt_column_list FROM STDIN  opt_header_list 
opt_seps opt_null_string opt_locked opt_best_effort opt_constraint 
opt_fwf_widths
+  | COPY opt_nr INTO qname opt_column_list FROM STDIN  opt_header_list 
opt_seps opt_null_string opt_locked opt_best_effort opt_constraint 
{ dlist *l = L();
  append_list(l, $4);
  append_list(l, $5);
@@ -2509,7 +2509,7 @@ copyfrom_stmt:
  append_int(l, $11);
  append_int(l, $12);
  append_int(l, $13);
- append_list(l, $14);
+ append_list(l, NULL);
  $$ = _symbol_create_list( SQL_COPYFROM, l ); }
| COPY opt_nr BINARY INTO qname FROM string_commalist /* binary copy from 
*/ opt_constraint
{ dlist *l = L();
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - fixed crash in fwf

2016-06-08 Thread Niels Nes

Changeset: 0610557e7ceb for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0610557e7ceb
Modified Files:
sql/backends/monet5/sql.c
Branch: fixed-width-format
Log Message:

fixed crash in fwf


diffs (12 lines):

diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3518,6 +3518,8 @@ mvc_import_table_wrap(Client cntxt, MalB
/* overwrite other delimiters to the ones the FWF 
stream uses */
sprintf((char*) tsep, "%c", STREAM_FWF_FIELD_SEP);
sprintf((char*) rsep, "%c", STREAM_FWF_RECORD_SEP);
+   if (!ssep) 
+   ssep = GDKmalloc(2);
ssep[0] = 0;
 
ss = stream_fwf_create(ss, ncol, widths, 
STREAM_FWF_FILLER);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - also allow for fix width if data c...

2016-06-08 Thread Niels Nes

Changeset: 507477d38dc4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=507477d38dc4
Modified Files:
sql/server/sql_parser.y
Branch: fixed-width-format
Log Message:

also allow for fix width if data comes from stdin.


diffs (20 lines):

diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -2497,7 +2497,7 @@ copyfrom_stmt:
  append_int(l, $13);
  append_list(l, $14);
  $$ = _symbol_create_list( SQL_COPYFROM, l ); }
-  | COPY opt_nr INTO qname opt_column_list FROM STDIN  opt_header_list 
opt_seps opt_null_string opt_locked opt_best_effort opt_constraint
+  | COPY opt_nr INTO qname opt_column_list FROM STDIN  opt_header_list 
opt_seps opt_null_string opt_locked opt_best_effort opt_constraint 
opt_fwf_widths
{ dlist *l = L();
  append_list(l, $4);
  append_list(l, $5);
@@ -2509,6 +2509,7 @@ copyfrom_stmt:
  append_int(l, $11);
  append_int(l, $12);
  append_int(l, $13);
+ append_list(l, $14);
  $$ = _symbol_create_list( SQL_COPYFROM, l ); }
| COPY opt_nr BINARY INTO qname FROM string_commalist /* binary copy from 
*/ opt_constraint
{ dlist *l = L();
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - make it compile with strict compil...

2016-06-08 Thread Niels Nes

Changeset: 03ae7c3004ed for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=03ae7c3004ed
Modified Files:
sql/backends/monet5/sql.c
sql/server/rel_updates.c
Branch: fixed-width-format
Log Message:

make it compile with strict compiler flags


diffs (30 lines):

diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3518,7 +3518,7 @@ mvc_import_table_wrap(Client cntxt, MalB
/* overwrite other delimiters to the ones the FWF 
stream uses */
sprintf((char*) tsep, "%c", STREAM_FWF_FIELD_SEP);
sprintf((char*) rsep, "%c", STREAM_FWF_RECORD_SEP);
-   sprintf((char*) ssep, "");
+   ssep[0] = 0;
 
ss = stream_fwf_create(ss, ncol, widths, 
STREAM_FWF_FILLER);
}
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1161,11 +1161,10 @@ rel_import(mvc *sql, sql_table *t, char 
if (fwf_widths && dlist_length(fwf_widths) > 0) {
dnode *dn;
int ncol = 0;
-   fwf_string = GDKmalloc(20 * dlist_length(fwf_widths) + 1); // a 
64 bit int needs 19 characters in decimal representation plus the separator
-   if (!fwf_string) {
+   char* fwf_string_cur = fwf_string = GDKmalloc(20 * 
dlist_length(fwf_widths) + 1); // a 64 bit int needs 19 characters in decimal 
representation plus the separator
+
+   if (!fwf_string) 
return NULL;
-   }
-   char* fwf_string_cur = fwf_string;
for (dn = fwf_widths->h; dn; dn = dn->next) {
fwf_string_cur += sprintf(fwf_string_cur, LLFMT"%c", 
dn->data.l_val, STREAM_FWF_FIELD_SEP);
ncol++;
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - merge with default

2016-06-08 Thread Hannes Muehleisen

Changeset: 07b23de27a1d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=07b23de27a1d
Modified Files:
configure.ag
monetdb5/extras/pyapi/pyapi.c
Branch: fixed-width-format
Log Message:

merge with default


diffs (29 lines):

diff --git a/configure.ag b/configure.ag
--- a/configure.ag
+++ b/configure.ag
@@ -2359,7 +2359,7 @@ if test "x$enable_pyintegration" != xno;
;;
esac
if test "x$have_pyconfig" = x; then
-   AC_PATH_PROG(PYCMD,python-config,,$XPATH)
+   AC_PATH_PROG(PYCMD,python-config,no,$PATH)
if test "x$PYCMD" = x; then
if test "x$enable_pyintegration" = xyes; then
AC_MSG_ERROR([python-config library required 
for Python integration support])
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -1620,10 +1620,10 @@ str
 MT_lock_set();
 if (!pyapiInitialized) {
 str msg = MAL_SUCCEED;
-char* iar = NULL;
 Py_Initialize();
-PyRun_SimpleString("import numpy");
-import_array1(iar);
+if (PyRun_SimpleString("import numpy") != 0 || _import_array() < 
0) {
+return PyError_CreateException("Failed to initialize embedded 
python", NULL);
+}
 msg = _connection_init();
 marshal_module = PyImport_Import(PyString_FromString("marshal"));
 if (marshal_module == NULL) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: fixed-width-format - Some cleanup before merge. Niels?

2016-06-08 Thread Hannes Muehleisen

Changeset: 769cc2e46a29 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=769cc2e46a29
Modified Files:
clients/R/Tests/copy_into_fwf.R
common/stream/stream.c
common/stream/stream.h
sql/backends/monet5/sql.c
sql/server/rel_updates.c
Branch: fixed-width-format
Log Message:

Some cleanup before merge. Niels?


diffs (90 lines):

diff --git a/clients/R/Tests/copy_into_fwf.R b/clients/R/Tests/copy_into_fwf.R
--- a/clients/R/Tests/copy_into_fwf.R
+++ b/clients/R/Tests/copy_into_fwf.R
@@ -23,7 +23,8 @@ if (dbExistsTable(con, "mtcars")) dbRemo
 dbBegin(con)
 dbSendQuery(con, "CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE 
PRECISION, disp DOUBLE PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, 
wt DOUBLE PRECISION, qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE 
PRECISION, gear DOUBLE PRECISION, carb DOUBLE PRECISION)")
 
-res <- dbSendQuery(con, paste0("COPY INTO mtcars FROM '", tf, "' FWF (4, 2, 6, 
4, 5, 6, 6, 2, 2, 2, 2)"))
+# delimiters are ineffective for fwf import just set them to make sure they 
dont break stuff
+res <- dbSendQuery(con, paste0("COPY OFFSET 1 INTO mtcars FROM '", tf, "' 
USING DELIMITERS 'a','b','c' NULL AS '' FWF (4, 2, 6, 4, 5, 6, 6, 2, 2, 2, 2)"))
 
 print(dbReadTable(con, "mtcars"))
 
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -4674,11 +4674,7 @@ stream * stream_blackhole_create (void)
 
 
 /* fixed-width format streams */
-
-#define STREAM_FWF_NAME "fwf"
-#define STREAM_FWF_FIELD_SEP '|'
-#define STREAM_FWF_ESCAPE '\\'
-#define STREAM_FWF_RECORD_SEP '\n'
+#define STREAM_FWF_NAME "fwf_ftw"
 
 typedef struct {
stream *s;
diff --git a/common/stream/stream.h b/common/stream/stream.h
--- a/common/stream/stream.h
+++ b/common/stream/stream.h
@@ -87,6 +87,12 @@ typedef __int128_t hge;
 #define ST_READ  0
 #define ST_WRITE 1
 
+/* fwf gets turned into a csv with these parameters */
+#define STREAM_FWF_FIELD_SEP '|'
+#define STREAM_FWF_ESCAPE '\\'
+#define STREAM_FWF_RECORD_SEP '\n'
+#define STREAM_FWF_FILLER ' '
+
 typedef struct stream stream;
 
 /* some os specific initialization */
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3501,17 +3501,26 @@ mvc_import_table_wrap(Client cntxt, MalB
}
widths = malloc(sizeof(size_t) * ncol);
if (!widths) {
-   // TODO: free other stuff
+   mnstr_destroy(ss);
+   GDKfree(tsep);
+   GDKfree(rsep);
+   GDKfree(ssep);
+   GDKfree(ns);
throw(MAL, "sql.copy_from", MAL_MALLOC_FAIL);
}
for (i = 0; i < width_len; i++) {
-   if (fixed_widths[i] == '|') {
+   if (fixed_widths[i] == STREAM_FWF_FIELD_SEP) {
fixed_widths[i] = '\0';
widths[current_width_entry++] = 
(size_t) atoll(val_start);
val_start = fixed_widths + i + 1;
}
}
-   ss = stream_fwf_create(ss, ncol, widths, ' ');
+   /* overwrite other delimiters to the ones the FWF 
stream uses */
+   sprintf((char*) tsep, "%c", STREAM_FWF_FIELD_SEP);
+   sprintf((char*) rsep, "%c", STREAM_FWF_RECORD_SEP);
+   sprintf((char*) ssep, "");
+
+   ss = stream_fwf_create(ss, ncol, widths, 
STREAM_FWF_FILLER);
}
 #if SIZEOF_VOID_P == 4
s = bstream_create(ss, 0x2);
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1167,7 +1167,7 @@ rel_import(mvc *sql, sql_table *t, char 
}
char* fwf_string_cur = fwf_string;
for (dn = fwf_widths->h; dn; dn = dn->next) {
-   fwf_string_cur += sprintf(fwf_string_cur, LLFMT"|", 
dn->data.l_val);
+   fwf_string_cur += sprintf(fwf_string_cur, LLFMT"%c", 
dn->data.l_val, STREAM_FWF_FIELD_SEP);
ncol++;
}
if(list_length(f->res) != ncol) {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

MonetDB: default - Reduce optimizer cost

MonetDB: iot - Finished stream creation through webserver with M...

MonetDB: Jun2016 - Typo.

MonetDB: default - approved output after fwf merge

MonetDB: default - Exports output

MonetDB: default - COPY INTO from fixed width format files

MonetDB: fixed-width-format - fwf is not supported on stdin (for...

MonetDB: fixed-width-format - fixed crash in fwf

MonetDB: fixed-width-format - also allow for fix width if data c...

MonetDB: fixed-width-format - make it compile with strict compil...

MonetDB: fixed-width-format - merge with default

MonetDB: fixed-width-format - Some cleanup before merge. Niels?

12 matches

Site Navigation

Mail list logo

Footer information