Changeset: dc746fc3538a for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dc746fc3538a Added Files: gdk/gdk_string.c Removed Files: monetdb5/modules/atoms/Tests/curl00.malC monetdb5/modules/atoms/Tests/curl00.stable.err monetdb5/modules/atoms/Tests/curl00.stable.out monetdb5/modules/atoms/mcurl.c monetdb5/modules/atoms/mcurl.mal sql/backends/monet5/vaults/Tests/vault00.malC Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 clients/Tests/exports.stable.out clients/mapiclient/Tests/mclient--help.stable.err clients/mapiclient/Tests/mclient--help.stable.err.Windows clients/mapiclient/mclient.1 clients/mapiclient/mclient.c clients/mapiclient/mhelp.c clients/odbc/driver/ODBCError.c configure.ag gdk/ChangeLog gdk/Makefile.ag gdk/gdk.h gdk/gdk_aggr.c gdk/gdk_atoms.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_calc.c gdk/gdk_calc_private.h gdk/gdk_hash.h gdk/gdk_join.c gdk/gdk_logger.c gdk/gdk_private.h gdk/gdk_select.c gdk/gdk_value.c monetdb5/modules/atoms/Makefile.ag monetdb5/modules/atoms/Tests/All monetdb5/modules/atoms/batxml.c monetdb5/modules/atoms/json.c monetdb5/modules/atoms/url.c monetdb5/modules/kernel/alarm.c monetdb5/modules/kernel/batmmath.c monetdb5/modules/kernel/mmath.c monetdb5/modules/mal/batcalc.c monetdb5/modules/mal/clients.c monetdb5/modules/mal/factories.c monetdb5/modules/mal/language.c monetdb5/modules/mal/mal_init.mal monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/mdb.c monetdb5/modules/mal/mkey.c sql/ChangeLog.Aug2018 sql/backends/monet5/UDF/pyapi/conversion.c sql/backends/monet5/sql.c sql/backends/monet5/sql_bat2time.c sql/backends/monet5/sql_cast.c sql/backends/monet5/sql_cast_impl_down_from_flt.h sql/backends/monet5/sql_cast_impl_int.h sql/backends/monet5/sql_cast_impl_up_to_flt.h sql/backends/monet5/sql_result.c sql/backends/monet5/sql_round_impl.h sql/backends/monet5/vaults/bam/bam_db_interface.c sql/backends/monet5/vaults/bam/bam_export.c sql/storage/bat/bat_storage.c sql/test/BugTracker-2012/Tests/url_script_test.Bug-2972.stable.err sql/test/BugTracker-2012/Tests/url_script_test.Bug-2972.stable.out sql/test/group-concat/Tests/groupconcat05.stable.out Branch: data-vaults Log Message:
Merge with default keeping the deleted vault directory We need to clean it up diffs (truncated from 6028 to 300 lines): diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -10552,10 +10552,6 @@ Ready. [ "mbr", "#read", "command mbr.#read():void ", "mbrREAD;", "" ] [ "mbr", "#tostr", "command mbr.#tostr():void ", "mbrTOSTR;", "" ] [ "mbr", "#write", "command mbr.#write():void ", "mbrWRITE;", "" ] -[ "mcurl", "deleterequest", "command mcurl.deleterequest(s:str):str ", "CURLdeleteRequest;", "Perform a http delete request" ] -[ "mcurl", "getrequest", "command mcurl.getrequest(s:str):str ", "CURLgetRequest;", "Perform a http get request" ] -[ "mcurl", "postrequest", "command mcurl.postrequest(s:str):str ", "CURLpostRequest;", "Perform a http post request" ] -[ "mcurl", "putrequest", "command mcurl.putrequest(s:str):str ", "CURLputRequest;", "Perform a http put request" ] [ "mdb", "List", "pattern mdb.List(M:str, F:str):void ", "MDBlist3Detail;", "Dump the routine M.F on standard out." ] [ "mdb", "List", "pattern mdb.List():void ", "MDBlistDetail;", "Dump the current routine on standard out." ] [ "mdb", "dump", "pattern mdb.dump():void ", "MDBdump;", "Dump instruction, stacktrace, and stack" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -14766,10 +14766,6 @@ Ready. [ "mbr", "#read", "command mbr.#read():void ", "mbrREAD;", "" ] [ "mbr", "#tostr", "command mbr.#tostr():void ", "mbrTOSTR;", "" ] [ "mbr", "#write", "command mbr.#write():void ", "mbrWRITE;", "" ] -[ "mcurl", "deleterequest", "command mcurl.deleterequest(s:str):str ", "CURLdeleteRequest;", "Perform a http delete request" ] -[ "mcurl", "getrequest", "command mcurl.getrequest(s:str):str ", "CURLgetRequest;", "Perform a http get request" ] -[ "mcurl", "postrequest", "command mcurl.postrequest(s:str):str ", "CURLpostRequest;", "Perform a http post request" ] -[ "mcurl", "putrequest", "command mcurl.putrequest(s:str):str ", "CURLputRequest;", "Perform a http put request" ] [ "mdb", "List", "pattern mdb.List(M:str, F:str):void ", "MDBlist3Detail;", "Dump the routine M.F on standard out." ] [ "mdb", "List", "pattern mdb.List():void ", "MDBlistDetail;", "Dump the current routine on standard out." ] [ "mdb", "dump", "pattern mdb.dump():void ", "MDBdump;", "Dump instruction, stacktrace, and stack" ] diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -118,7 +118,7 @@ gdk_return BATextend(BAT *b, BUN newcap) void BATfakeCommit(BAT *b); gdk_return BATfirstn(BAT **topn, BAT **gids, BAT *b, BAT *cands, BAT *grps, BUN n, bool asc, bool distinct) __attribute__((__warn_unused_result__)); int BATgetaccess(BAT *b); -PROPrec *BATgetprop(BAT *b, int idx); +PROPrec *BATgetprop(BAT *b, enum prop_t idx); gdk_return BATgroup(BAT **groups, BAT **extents, BAT **histo, BAT *b, BAT *s, BAT *g, BAT *e, BAT *h) __attribute__((__warn_unused_result__)); const char *BATgroupaggrinit(BAT *b, BAT *g, BAT *e, BAT *s, oid *minp, oid *maxp, BUN *ngrpp, BUN *startp, BUN *endp, const oid **candp, const oid **candendp); gdk_return BATgroupavg(BAT **bnp, BAT **cntsp, BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error, int scale); @@ -161,7 +161,7 @@ BAT *BATproject(BAT *l, BAT *r); BAT *BATprojectchain(BAT **bats); gdk_return BATrangejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *rl, BAT *rh, BAT *sl, BAT *sr, bool li, bool hi, BUN estimate) __attribute__((__warn_unused_result__)); gdk_return BATreplace(BAT *b, BAT *p, BAT *n, bool force) __attribute__((__warn_unused_result__)); -void BATrmprop(BAT *b, int idx); +void BATrmprop(BAT *b, enum prop_t idx); gdk_return BATroles(BAT *b, const char *tnme); BAT *BATsample(BAT *b, BUN n); BAT *BATsample_with_seed(BAT *b, BUN n, unsigned seed); @@ -170,7 +170,7 @@ gdk_return BATsemijoin(BAT **r1p, BAT ** gdk_return BATsetaccess(BAT *b, int mode); void BATsetcapacity(BAT *b, BUN cnt); void BATsetcount(BAT *b, BUN cnt); -void BATsetprop(BAT *b, int idx, int type, const void *v); +void BATsetprop(BAT *b, enum prop_t idx, int type, const void *v); BAT *BATslice(BAT *b, BUN low, BUN high); gdk_return BATsort(BAT **sorted, BAT **order, BAT **groups, BAT *b, BAT *o, BAT *g, bool reverse, bool stable) __attribute__((__warn_unused_result__)); gdk_return BATstr_group_concat(ValPtr res, BAT *b, BAT *s, bool skip_nils, bool abort_on_error, bool nil_if_empty, const char *separator); @@ -1180,10 +1180,6 @@ str CMDvarSUBsignal(Client cntxt, MalBlk str CMDvarXOR(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str COPYrejects(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); str COPYrejects_clear(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci); -str CURLdeleteRequest(str *retval, str *url); -str CURLgetRequest(str *retval, str *url); -str CURLpostRequest(str *retval, str *url); -str CURLputRequest(str *retval, str *url); str FCTgetArrival(bat *ret); str FCTgetCaller(int *ret); str FCTgetDeparture(bat *ret); diff --git a/clients/mapiclient/Tests/mclient--help.stable.err b/clients/mapiclient/Tests/mclient--help.stable.err --- a/clients/mapiclient/Tests/mclient--help.stable.err +++ b/clients/mapiclient/Tests/mclient--help.stable.err @@ -35,6 +35,7 @@ Options are: SQL specific opions -n nullstr | --null=nullstr change NULL representation for sql, csv and tab output modes -a | --autocommit turn off autocommit mode + -R | --allow-remote allow remote content -r nr | --rows=nr for pagination -w nr | --width=nr for pagination -D | --dump create an SQL dump diff --git a/clients/mapiclient/Tests/mclient--help.stable.err.Windows b/clients/mapiclient/Tests/mclient--help.stable.err.Windows --- a/clients/mapiclient/Tests/mclient--help.stable.err.Windows +++ b/clients/mapiclient/Tests/mclient--help.stable.err.Windows @@ -34,6 +34,7 @@ Options are: SQL specific opions -n nullstr | --null=nullstr change NULL representation for sql, csv and tab output modes -a | --autocommit turn off autocommit mode + -R | --allow-remote allow remote content -r nr | --rows=nr for pagination -w nr | --width=nr for pagination -D | --dump create an SQL dump diff --git a/clients/mapiclient/mclient.1 b/clients/mapiclient/mclient.1 --- a/clients/mapiclient/mclient.1 +++ b/clients/mapiclient/mclient.1 @@ -266,6 +266,17 @@ string, use \fB\-n \(dq\(dq\fP (with the Switch autocommit mode off. By default, autocommit mode is on. .TP +\fB\-\-allow\-remote\fP (\fB\-R\fP) +Allow remote content (URLs) in the +.B COPY INTO +.I table +.B FROM +.I file +.B ON CLIENT +\&... +query. Remote content is retrieved by +.IR mclient . +.TP \fB\-\-rows=\fP\fInr\fP (\fB\-r\fP \fInr\fP) If specified, query results will be paged by an internal pager at the specified number of lines. @@ -459,6 +470,20 @@ in the CSV file. This, because otherwise the remainder of the file will be interpreted as SQL queries. .PP +Another, easier method to have the client read the file content is as +follows. +.PP +mclient \-d $db \-s \(dqCOPY INTO $table FROM '$file' ON CLIENT USING DELIMITERS ',','\e\en',\e\(dq'\(dq +.PP +If, in addition, the option +.B \-\-allow\-remote +is passed to +.IR mclient , +the +.I $file +in the above query can also be a URL. It then has to have the form +.IB schema :// string . +.PP See https://www.monetdb.org/Documentation/Manuals/SQLreference/CopyInto for more information about the COPY INTO query. .SH SEE ALSO diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c --- a/clients/mapiclient/mclient.c +++ b/clients/mapiclient/mclient.c @@ -76,6 +76,7 @@ static bool echoquery = false; static char *encoding; #endif static bool errseen = false; +static bool allow_remote = false; #define setPrompt() sprintf(promptbuf, "%.*s>", (int) sizeof(promptbuf) - 2, language) #define debugMode() (strncmp(promptbuf, "mdb", 3) == 0) @@ -3035,6 +3036,9 @@ struct privdata { #define READSIZE (1 << 16) //#define READSIZE (1 << 20) +static char alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + static char * getfile(void *data, const char *filename, bool binary, uint64_t offset, size_t *size) @@ -3044,12 +3048,11 @@ getfile(void *data, const char *filename struct privdata *priv = data; ssize_t s; + *size = 0; /* most returns require this */ if (priv->buf == NULL) { priv->buf = malloc(READSIZE); - if (priv->buf == NULL) { - *size = 0; - return "allocation failed"; - } + if (priv->buf == NULL) + return "allocation failed in client"; } buf = priv->buf; if (filename != NULL) { @@ -3059,8 +3062,22 @@ getfile(void *data, const char *filename offset = 0; } else { f = open_rastream(filename); + if (f == NULL) { + size_t x; + /* simplistic check for URL + * (schema://...) */ + if ((x = strspn(filename, alpha)) > 0 + && filename[x] == ':' + && filename[x+1] == '/' + && filename[x+2] == '/') { + if (allow_remote) + f = open_urlstream(filename); + else + return "client refuses to retrieve remote content"; + } + } #ifdef HAVE_ICONV - if (encoding) { + else if (encoding) { stream *tmpf = f; f = iconv_rstream(f, encoding, mnstr_name(f)); if (f == NULL) @@ -3068,21 +3085,17 @@ getfile(void *data, const char *filename } #endif } - if (f == NULL) { - *size = 0; /* indicate error */ + if (f == NULL) return "cannot open file"; - } while (offset > 1) { s = mnstr_readline(f, buf, READSIZE); if (s < 0) { close_stream(f); - *size = 0; return "error reading file"; } if (s == 0) { /* reached EOF withing offset lines */ close_stream(f); - *size = 0; return NULL; } if (buf[s - 1] == '\n') @@ -3100,7 +3113,6 @@ getfile(void *data, const char *filename } s = mnstr_read(f, buf, 1, READSIZE); if (s <= 0) { - *size = 0; close_stream(f); priv->f = NULL; return s < 0 ? "error reading file" : NULL; @@ -3183,6 +3195,7 @@ usage(const char *prog, int xit) fprintf(stderr, "\nSQL specific opions \n"); fprintf(stderr, " -n nullstr | --null=nullstr change NULL representation for sql, csv and tab output modes\n"); fprintf(stderr, " -a | --autocommit turn off autocommit mode\n"); + fprintf(stderr, " -R | --allow-remote allow remote content\n"); fprintf(stderr, " -r nr | --rows=nr for pagination\n"); fprintf(stderr, " -w nr | --width=nr for pagination\n"); fprintf(stderr, " -D | --dump create an SQL dump\n"); @@ -3247,6 +3260,7 @@ main(int argc, char **argv) {"width", 1, 0, 'w'}, {"Xdebug", 0, 0, 'X'}, {"timezone", 0, 0, 'z'}, + {"allow-remote", 0, 0, 'R'}, {0, 0, 0, 0} }; @@ -3289,7 +3303,7 @@ main(int argc, char **argv) #ifdef HAVE_ICONV "E:" #endif - "f:h:Hil:L:n:Np:P:r:s:t:u:vw:Xz" + "f:h:Hil:L:n:Np:P:r:Rs:t:u:vw:Xz" #ifdef HAVE_POPEN "|:" #endif @@ -3297,12 +3311,8 @@ main(int argc, char **argv) long_options, &option_index)) != -1) { switch (c) { case 0: -#ifdef HAVE_POPEN - if (strcmp(long_options[option_index].name, "pager") == 0) { - pager = optarg; - (void) pager; /* will be further used later */ - } -#endif + /* only needed for options that only have a + * long form */ break; case 'a': autocommit = false; @@ -3390,6 +3400,9 @@ main(int argc, char **argv) assert(optarg); rowsperpage = atoi(optarg); break; + case 'R': + allow_remote = true; + break; case 's': assert(optarg); command = optarg; diff --git a/clients/mapiclient/mhelp.c b/clients/mapiclient/mhelp.c --- a/clients/mapiclient/mhelp.c +++ b/clients/mapiclient/mhelp.c @@ -115,16 +115,16 @@ SQLhelp sqlhelp[] = { NULL}, {"COPY BINARY", "Append binary representations into a table", - "COPY [nrofrecords] BINARY INTO qname [column_list] FROM string [',' ...] [NO CONSTRAINT]", + "COPY [nrofrecords] BINARY INTO qname [column_list] FROM string [',' ...] [ON { CLIENT | SERVER }] [NO CONSTRAINT]", "nrofrecords", "See also https://www.monetdb.org/Documentation/Cookbooks/SQLrecipes/BinaryBulkLoad"}, {"COPY INTO", - "Parse a csv-file into a table", - "COPY [nrofrecords] INTO qname [column_list] FROM string [',' ...] [headerlist] [ separators]\n" + "Parse a csv file into a table or write a query result to a csv file", + "COPY [nrofrecords] INTO qname [column_list] FROM string [',' ...] [headerlist] [ON { CLIENT | SERVER }] [ separators]\n" " [NULL [AS] string] [LOCKED] [BEST EFFORT] [NO CONSTRAINT] [FWF '(' integer [',' ...] ')'\n" "COPY [nrofrecords] INTO qname [column_list] FROM STDIN [headerlist] [ separators]\n" " [NULL [AS] string] [LOCKED] [BEST EFFORT] [NO CONSTRAINT]\n" - "COPY query_expression INTO [STDOUT | string] [seps] [NULL [AS] string]", + "COPY query_expression INTO [STDOUT | string [ON { CLIENT | SERVER }]] [seps] [NULL [AS] string]", "nrofrecords,headerlist,separators", "See also https://www.monetdb.org/Documentation/Cookbooks/SQLrecipes/LoadingBulkData"}, {"COPY LOADER", diff --git a/clients/odbc/driver/ODBCError.c b/clients/odbc/driver/ODBCError.c _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
