Changeset: f45e2f17db1d for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f45e2f17db1d
Modified Files:
clients/Tests/All
clients/Tests/SQL-dump_geom.stable.out.32bit
clients/mapiclient/mclient.c
clients/mapilib/mapi.rc
clients/odbc/driver/driver.rc
clients/odbc/winsetup/setup.rc
common/stream/stream.c
gdk/libbat.rc
monetdb5/tools/libmonetdb5.rc
sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
sql/benchmarks/tpch/Tests/22-explain.stable.out.32bit
sql/jdbc/tests/Tests/All
sql/test/BugTracker-2010/Tests/unicode-bom.Bug-2641.sql
sql/test/BugTracker-2010/Tests/unicode-bom.Bug-2641.stable.err
sql/test/BugTracker-2010/Tests/unicode-bom.Bug-2641.stable.out
sql/test/BugTracker-2011/Tests/All
sql/test/BugTracker-2012/Tests/All
sql/test/Dependencies/Tests/All
sql/test/Tests/All
sql/test/bugs/Tests/All
sql/test/leaks/Tests/All
Branch: default
Log Message:
Merge with Oct2014 branch.
diffs (truncated from 821 to 300 lines):
diff --git a/clients/Tests/All b/clients/Tests/All
--- a/clients/Tests/All
+++ b/clients/Tests/All
@@ -1,9 +1,9 @@
exports
-HAVE_CFITSIO&HAVE_GEOM&HAVE_GSL&HAVE_LIBR&HAVE_SAMTOOLS&HAVE_SPHINXCLIENT?MAL-signatures_all
-HAVE_CFITSIO&HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT?MAL-signatures_fits_geom
-!HAVE_CFITSIO&HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT?MAL-signatures_geom
-!HAVE_CFITSIO&!HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT?MAL-signatures_none
-HAVE_GEOM&HAVE_GSL&HAVE_SAMTOOLS?SQL-dump_all
-HAVE_GEOM&!HAVE_GSL&!HAVE_SAMTOOLS?SQL-dump_geom
-!HAVE_GEOM&!HAVE_GSL&!HAVE_SAMTOOLS?SQL-dump_none
+HAVE_CFITSIO&HAVE_GEOM&HAVE_GSL&HAVE_LIBR&HAVE_SAMTOOLS&HAVE_SPHINXCLIENT&!ENABLE_DATACELL?MAL-signatures_all
+HAVE_CFITSIO&HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT&!ENABLE_DATACELL?MAL-signatures_fits_geom
+!HAVE_CFITSIO&HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT&!ENABLE_DATACELL?MAL-signatures_geom
+!HAVE_CFITSIO&!HAVE_GEOM&!HAVE_GSL&!HAVE_LIBR&!HAVE_SAMTOOLS&!HAVE_SPHINXCLIENT&!ENABLE_DATACELL?MAL-signatures_none
+HAVE_GEOM&HAVE_GSL&HAVE_SAMTOOLS&!ENABLE_DATACELL?SQL-dump_all
+HAVE_GEOM&!HAVE_GSL&!HAVE_SAMTOOLS&!ENABLE_DATACELL?SQL-dump_geom
+!HAVE_GEOM&!HAVE_GSL&!HAVE_SAMTOOLS&!ENABLE_DATACELL?SQL-dump_none
MERCURIAL?malcheck
diff --git a/clients/mapiclient/mclient.c b/clients/mapiclient/mclient.c
--- a/clients/mapiclient/mclient.c
+++ b/clients/mapiclient/mclient.c
@@ -1796,9 +1796,9 @@ doFileBulk(Mapi mid, FILE *fp)
size_t length;
MapiHdl hdl = mapi_get_active(mid);
MapiMsg rc = MOK;
- int bufsize = 0;
+ size_t bufsize = 0;
int first = 1; /* first line processing */
- size_t skip;
+ char *input_encoding = encoding;
bufsize = BLOCK - 1;
buf = malloc(bufsize + 1);
@@ -1821,8 +1821,51 @@ doFileBulk(Mapi mid, FILE *fp)
if (hdl == NULL)
break; /* nothing more to do */
} else {
+ if (first &&
+ length >= UTF8BOMLENGTH &&
+ strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) {
+ input_encoding = NULL;
+ memmove(buf, buf + UTF8BOMLENGTH,
+ length -= UTF8BOMLENGTH);
+ }
+ first = 0;
+ if (input_encoding != NULL && cd_in != (iconv_t) -1) {
+ ICONV_CONST char *from;
+ size_t fromlen;
+ char *to, *nbuf;
+ size_t tolen = length * 4 + 1;
+
+ if (tolen <= bufsize)
+ tolen = bufsize + 1;
+ retry:
+ from = buf;
+ fromlen = length;
+ nbuf = to = malloc(tolen);
+ if (iconv(cd_in, &from, &fromlen, &to, &tolen)
== (size_t) -1) {
+ switch (errno) {
+ case EILSEQ:
+ free(nbuf);
+ free(buf);
+ fprintf(stderr, "Illegal input
sequence\n");
+ return 1;
+ case E2BIG:
+ tolen *= 2;
+ free(nbuf);
+ goto retry;
+ case EINVAL:
+ default:
+ /* shouldn't happen */
+ free(nbuf);
+ free(buf);
+ return 1;
+ }
+ }
+ length = tolen;
+ free(buf);
+ buf = nbuf;
+ }
buf[length] = 0;
- if (strchr(buf, '\0') < buf + length) {
+ if (strlen(buf) < length) {
fprintf(stderr, "NULL byte in input\n");
errseen = 1;
break;
@@ -1835,23 +1878,14 @@ doFileBulk(Mapi mid, FILE *fp)
CHECK_RESULT(mid, hdl, buf, continue, buf);
}
- if (first &&
- length >= UTF8BOMLENGTH &&
- strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0)
- skip = UTF8BOMLENGTH; /* skip Byte Order Mark (BOM) */
- else
- skip = 0;
- first = 0;
- if (length > skip) {
- assert(hdl != NULL);
+ assert(hdl != NULL);
- mapi_query_part(hdl, buf + skip, length - skip);
- CHECK_RESULT(mid, hdl, buf + skip, continue, buf);
+ mapi_query_part(hdl, buf, length);
+ CHECK_RESULT(mid, hdl, buf, continue, buf);
- /* make sure there is a newline in the buffer */
- if (strchr(buf + skip, '\n') == NULL)
- continue;
- }
+ /* make sure there is a newline in the buffer */
+ if (strchr(buf, '\n') == NULL)
+ continue;
assert(hdl != NULL);
/* If the server wants more but we're at the end of
@@ -1865,14 +1899,14 @@ doFileBulk(Mapi mid, FILE *fp)
(length > 0 || mapi_query_done(hdl) == MMORE))
continue; /* get more data */
- CHECK_RESULT(mid, hdl, buf + skip, continue, buf);
+ CHECK_RESULT(mid, hdl, buf, continue, buf);
rc = format_result(mid, hdl, 0);
if (rc == MMORE && (length > 0 || mapi_query_done(hdl) != MOK))
continue; /* get more data */
- CHECK_RESULT(mid, hdl, buf + skip, continue, buf);
+ CHECK_RESULT(mid, hdl, buf, continue, buf);
mapi_close_handle(hdl);
hdl = NULL;
@@ -1951,6 +1985,9 @@ doFile(Mapi mid, const char *file, int u
FILE *fp;
char *prompt = NULL;
int prepno = 0;
+#ifdef HAVE_ICONV
+ char *input_encoding = encoding;
+#endif
(void) save_history; /* not used if no readline */
if (strcmp(file, "-") == 0) {
@@ -2059,11 +2096,27 @@ doFile(Mapi mid, const char *file, int u
else {
*line = 0;
line = buf;
+ /* if we successfully read a line, and
+ * it was the first one, look whether
+ * it starts with the UTF-8 encoding
+ * of the Unicode BOM, and if so, skip
+ * the BOM and make sure we treat the
+ * rest of the file as UTF-8, no
+ * matter what encoding the user has
+ * specified */
+ if (lineno == 1 &&
+ strncmp(line, UTF8BOM, UTF8BOMLENGTH) == 0)
{
+ line += UTF8BOMLENGTH;
+#ifdef HAVE_ICONV
+ input_encoding = NULL;
+#endif
+ }
}
}
#ifdef HAVE_ICONV
- if (line != NULL && fp == stdin &&
- encoding != NULL && cd_in != (iconv_t) -1) {
+ if (line != NULL &&
+ input_encoding != NULL &&
+ cd_in != (iconv_t) -1) {
ICONV_CONST char *from = line;
size_t fromlen = strlen(from);
int factor = 4;
@@ -2104,15 +2157,8 @@ doFile(Mapi mid, const char *file, int u
if (oldbuf == NULL)
free(buf);
buf = line;
- } else
+ }
#endif
- if (line != NULL &&
-#ifdef HAVE_ICONV
- encoding == NULL &&
-#endif
- lineno == 1 &&
- strncmp(line, UTF8BOM, UTF8BOMLENGTH) == 0)
- line += UTF8BOMLENGTH; /* skip Byte Order Mark (BOM) */
lineno++;
if (line == NULL) {
/* end of file */
diff --git a/clients/mapilib/mapi.rc b/clients/mapilib/mapi.rc
--- a/clients/mapilib/mapi.rc
+++ b/clients/mapilib/mapi.rc
@@ -23,7 +23,7 @@ BEGIN
// Maintained via vertoo. Please don't modify by hand!
// Contact monetdb-develop...@lists.sourceforge.net for details and/or
assistance.
VALUE "InternalName", "Mapi\0"
- VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2014\0"
+ VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2015\0"
VALUE "LegalTrademarks", "\0"
VALUE "OriginalFilename", "Mapi.dll\0"
VALUE "PrivateBuild", "\0"
diff --git a/clients/odbc/driver/driver.rc b/clients/odbc/driver/driver.rc
--- a/clients/odbc/driver/driver.rc
+++ b/clients/odbc/driver/driver.rc
@@ -23,7 +23,7 @@ BEGIN
// Maintained via vertoo. Please don't modify by hand!
// Contact monetdb-develop...@lists.sourceforge.net for details and/or
assistance.
VALUE "InternalName", "libMonetODBC\0"
- VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2014\0"
+ VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2015\0"
VALUE "LegalTrademarks", "\0"
VALUE "OriginalFilename", "libMonetODBC.dll\0"
VALUE "PrivateBuild", "\0"
diff --git a/clients/odbc/winsetup/setup.rc b/clients/odbc/winsetup/setup.rc
--- a/clients/odbc/winsetup/setup.rc
+++ b/clients/odbc/winsetup/setup.rc
@@ -82,7 +82,7 @@ BEGIN
VALUE "FileDescription", "MonetDB ODBC Setup DLL"
VALUE "FileVersion", "11.20.0"
VALUE "InternalName", "libMonetODBCs.dll"
- VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2014"
+ VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2015"
VALUE "OriginalFilename", "libMonetODBCs.dll"
VALUE "ProductName", "MonetDB SQL Server"
VALUE "ProductVersion", "11.20.0"
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -896,6 +896,15 @@ open_gzstream(const char *filename, cons
s->close = stream_gzclose;
s->flush = stream_gzflush;
s->stream_data.p = (void *) fp;
+ if (flags[0] == 'r') {
+ char buf[UTF8BOMLENGTH];
+ if (gzread(fp, buf, UTF8BOMLENGTH) == UTF8BOMLENGTH &&
+ strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) {
+ s->isutf8 = 1;
+ } else {
+ gzrewind(fp);
+ }
+ }
return s;
}
@@ -1092,11 +1101,23 @@ open_bzstream(const char *filename, cons
s->flush = NULL;
s->stream_data.p = (void *) bzp;
if (strchr(flags, 'r') != NULL) {
+ s->access = ST_READ;
bzp->b = BZ2_bzReadOpen(&err, bzp->f, 0, 0, NULL, 0);
- s->access = ST_READ;
if (err == BZ_STREAM_END) {
BZ2_bzReadClose(&err, bzp->b);
bzp->b = NULL;
+ } else {
+ char buf[UTF8BOMLENGTH];
+
+ if (stream_bzread(s, buf, 1, UTF8BOMLENGTH) ==
UTF8BOMLENGTH &&
+ strncmp(buf, UTF8BOM, UTF8BOMLENGTH) == 0) {
+ s->isutf8 = 1;
+ } else if (s->stream_data.p) {
+ bzp = s->stream_data.p;
+ BZ2_bzReadClose(&err, bzp->b);
+ rewind(bzp->f);
+ bzp->b = BZ2_bzReadOpen(&err, bzp->f, 0, 0,
NULL, 0);
+ }
}
} else {
bzp->b = BZ2_bzWriteOpen(&err, bzp->f, 9, 0, 30);
diff --git a/gdk/libbat.rc b/gdk/libbat.rc
--- a/gdk/libbat.rc
+++ b/gdk/libbat.rc
@@ -23,7 +23,7 @@ BEGIN
// Maintained via vertoo. Please don't modify by hand!
// Contact monetdb-develop...@lists.sourceforge.net for details and/or
assistance.
VALUE "InternalName", "libbat\0"
- VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2014\0"
+ VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2015\0"
VALUE "LegalTrademarks", "\0"
VALUE "OriginalFilename", "libbat.dll\0"
VALUE "PrivateBuild", "\0"
diff --git a/monetdb5/tools/libmonetdb5.rc b/monetdb5/tools/libmonetdb5.rc
--- a/monetdb5/tools/libmonetdb5.rc
+++ b/monetdb5/tools/libmonetdb5.rc
@@ -23,7 +23,7 @@ BEGIN
// Maintained via vertoo. Please don't modify by hand!
// Contact monetdb-develop...@lists.sourceforge.net for details and/or
assistance.
VALUE "InternalName", "libmonetdb5\0"
- VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2014\0"
+ VALUE "LegalCopyright", "Copyright © MonetDB B.V. 2008-2015\0"
VALUE "LegalTrademarks", "\0"
VALUE "OriginalFilename", "libmonetdb5.dll\0"
VALUE "PrivateBuild", "\0"
diff --git a/sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
b/sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
--- a/sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
+++ b/sql/benchmarks/tpch/Tests/20-explain.stable.out.32bit
@@ -94,12 +94,12 @@ function user.s2_1{autoCommit=true}(A0:s
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
