Changeset: 14adcbd67a10 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=14adcbd67a10 Modified Files: clients/Tests/MAL-signatures.stable.out clients/Tests/MAL-signatures.stable.out.int128 gdk/gdk_batop.c gdk/gdk_project.c gdk/gdk_select.c monetdb5/mal/mal_runtime.c monetdb5/modules/atoms/blob.c sql/backends/monet5/rel_bin.c sql/server/rel_optimizer.c testing/Mtest.py.in Branch: unlock Log Message:
merged with default diffs (truncated from 4325 to 300 lines): diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -796,3 +796,5 @@ 929f5e280bc1532a2bfaab127ca7915dc3b69a33 742b7847cfdcea39a6c19ab29eb35471d46bb2bb Oct2020_11 929f5e280bc1532a2bfaab127ca7915dc3b69a33 Oct2020_SP2_release 742b7847cfdcea39a6c19ab29eb35471d46bb2bb Oct2020_SP2_release +17d27ad30941c81e4bc700300912e84e9b9a8c37 Oct2020_13 +17d27ad30941c81e4bc700300912e84e9b9a8c37 Oct2020_SP3_release diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -84,7 +84,7 @@ Group: Applications/Databases License: MPLv2.0 URL: https://www.monetdb.org/ BugURL: https://bugs.monetdb.org/ -Source: https://www.monetdb.org/downloads/sources/Oct2020-SP2/%{name}-%{version}.tar.bz2 +Source: https://www.monetdb.org/downloads/sources/Oct2020-SP3/%{name}-%{version}.tar.bz2 # The Fedora packaging document says we need systemd-rpm-macros for # the _unitdir and _tmpfilesdir macros to exist; however on RHEL 7 @@ -828,6 +828,10 @@ else fi %changelog +* Thu Feb 11 2021 Sjoerd Mullender <sjo...@acm.org> - 11.39.13-20210211 +- Rebuilt. +- GH#7049: Implement DISTINCT for GROUP_CONCAT + * Mon Jan 18 2021 Sjoerd Mullender <sjo...@acm.org> - 11.39.11-20210118 - Rebuilt. - GH#3772: Any user can grant a role. diff --git a/clients/Tests/MAL-signatures.stable.out b/clients/Tests/MAL-signatures.stable.out --- a/clients/Tests/MAL-signatures.stable.out +++ b/clients/Tests/MAL-signatures.stable.out @@ -3818,6 +3818,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:sht], X_2:bat[:oid]):bat[:bit] ", "CMDconvert_bit;", "" ] [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:str]):bat[:bit] ", "CMDconvert_bit;", "" ] [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:str], X_2:bat[:oid]):bat[:bit] ", "CMDconvert_bit;", "" ] +[ "batcalc", "blob", "command batcalc.blob(X_1:bat[:blob], X_2:bat[:oid]):bat[:blob] ", "BLOBblob_blob_bulk;", "" ] [ "batcalc", "blob", "command batcalc.blob(X_1:bat[:str], X_2:bat[:oid]):bat[:blob] ", "batstr_2_blob;", "" ] [ "batcalc", "bte", "pattern batcalc.bte(X_1:bat[:bit]):bat[:bte] ", "CMDconvertsignal_bte;", "" ] [ "batcalc", "bte", "pattern batcalc.bte(X_1:bat[:bit], X_2:bat[:oid]):bat[:bte] ", "CMDconvertsignal_bte;", "" ] diff --git a/clients/Tests/MAL-signatures.stable.out.int128 b/clients/Tests/MAL-signatures.stable.out.int128 --- a/clients/Tests/MAL-signatures.stable.out.int128 +++ b/clients/Tests/MAL-signatures.stable.out.int128 @@ -5442,6 +5442,7 @@ stdout of test 'MAL-signatures` in direc [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:sht], X_2:bat[:oid]):bat[:bit] ", "CMDconvert_bit;", "" ] [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:str]):bat[:bit] ", "CMDconvert_bit;", "" ] [ "batcalc", "bit_noerror", "pattern batcalc.bit_noerror(X_1:bat[:str], X_2:bat[:oid]):bat[:bit] ", "CMDconvert_bit;", "" ] +[ "batcalc", "blob", "command batcalc.blob(X_1:bat[:blob], X_2:bat[:oid]):bat[:blob] ", "BLOBblob_blob_bulk;", "" ] [ "batcalc", "blob", "command batcalc.blob(X_1:bat[:str], X_2:bat[:oid]):bat[:blob] ", "batstr_2_blob;", "" ] [ "batcalc", "bte", "pattern batcalc.bte(X_1:bat[:bit]):bat[:bte] ", "CMDconvertsignal_bte;", "" ] [ "batcalc", "bte", "pattern batcalc.bte(X_1:bat[:bit], X_2:bat[:oid]):bat[:bte] ", "CMDconvertsignal_bte;", "" ] diff --git a/clients/mapiclient/dump.c b/clients/mapiclient/dump.c --- a/clients/mapiclient/dump.c +++ b/clients/mapiclient/dump.c @@ -882,6 +882,34 @@ dump_column_definition(Mapi mid, stream mnstr_printf(toConsole, "\t"); space = dquoted_print(toConsole, c_name, " "); mnstr_printf(toConsole, "%*s", CAP(slen - space), ""); + if (s != NULL && t != NULL && + strcmp(c_type, "char") == 0 && strcmp(c_type_digits, "0") == 0) { + /* if the number of characters is not specified (due to a bug), + * calculate a size */ + char *c = descape(c_name); + if (c != NULL) { + size_t qlen = strlen(c) + strlen(s) + strlen(t) + 64; + char *q = malloc(qlen); + if (q != NULL) { + snprintf(q, qlen, "SELECT max(length(\"%s\")) FROM \"%s\".\"%s\"", c, s, t); + MapiHdl h = mapi_query(mid, q); + if (h != NULL) { + if (mapi_fetch_row(h) != 0) { + const char *d = mapi_fetch_field(h, 0); + free(c_type_digits); + /* if NULL, i.e. no non-NULL values, fill in 1 */ + c_type_digits = strdup(d ? d : "1"); + fprintf(stderr, "Warning: fixing size of CHAR column for %s of table %s.%s\n", c_name, schema, tname); + } + mapi_close_handle(h); + } + free(q); + } + free(c); + } + if (c_type_digits == NULL) + goto bailout; + } space = dump_type(mid, toConsole, c_type, c_type_digits, c_type_scale, hashge); if (strcmp(c_null, "false") == 0) { mnstr_printf(toConsole, "%*s NOT NULL", @@ -1256,7 +1284,11 @@ describe_table(Mapi mid, const char *sch goto bailout; while (mapi_fetch_row(hdl) != 0) expr = mapi_fetch_field(hdl, 0); - mnstr_printf(toConsole, " PARTITION BY %s %s (%s)", phow, pusing, expr); + mnstr_printf(toConsole, " PARTITION BY %s %s (", phow, pusing); + if (column) + dquoted_print(toConsole, expr, ")"); + else + mnstr_printf(toConsole, "%s)", expr); mapi_close_handle(hdl); } } diff --git a/cmake/monetdb-versions.cmake b/cmake/monetdb-versions.cmake --- a/cmake/monetdb-versions.cmake +++ b/cmake/monetdb-versions.cmake @@ -50,7 +50,7 @@ set(MONETDB_VERSION "${MONETDB_VERSION_M # version of the GDK library (subdirectory gdk; also includes # common/options and common/utils) set(GDK_VERSION_CURRENT "22") -set(GDK_VERSION_REVISION "1") +set(GDK_VERSION_REVISION "2") set(GDK_VERSION_AGE "1") math(EXPR GDK_VERSION_MAJOR "${GDK_VERSION_CURRENT} - ${GDK_VERSION_AGE}") set(GDK_VERSION "${GDK_VERSION_MAJOR}.${GDK_VERSION_AGE}.${GDK_VERSION_REVISION}") @@ -64,7 +64,7 @@ set(MAPI_VERSION "${MAPI_VERSION_MAJOR}. # version of the MONETDB5 library (subdirectory monetdb5, not including extras or sql) set(MONETDB5_VERSION_CURRENT "30") -set(MONETDB5_VERSION_REVISION "4") +set(MONETDB5_VERSION_REVISION "5") set(MONETDB5_VERSION_AGE "0") math(EXPR MONETDB5_VERSION_MAJOR "${MONETDB5_VERSION_CURRENT} - ${MONETDB5_VERSION_AGE}") set(MONETDB5_VERSION "${MONETDB5_VERSION_MAJOR}.${MONETDB5_VERSION_AGE}.${MONETDB5_VERSION_REVISION}") diff --git a/common/stream/Tests/All b/common/stream/Tests/All --- a/common/stream/Tests/All +++ b/common/stream/Tests/All @@ -14,3 +14,7 @@ HAVE_CURL?urlstream read_iconv write_iconv +HAVE_LIBBZ2?read_concatenated_bz2 +HAVE_LIBZ?read_concatenated_gz +HAVE_LIBLZ4&HAVE_PYTHON_LZ4?read_concatenated_lz4 +HAVE_LIBLZMA?read_concatenated_xz diff --git a/common/stream/Tests/read_concatenated_bz2.py b/common/stream/Tests/read_concatenated_bz2.py new file mode 100644 --- /dev/null +++ b/common/stream/Tests/read_concatenated_bz2.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys, os +sys.path.append(os.environ.get('TSTSRCDIR','.')) +import read_tests + +read_tests.read_concatenated('bz2') diff --git a/common/stream/Tests/read_concatenated_gz.py b/common/stream/Tests/read_concatenated_gz.py new file mode 100644 --- /dev/null +++ b/common/stream/Tests/read_concatenated_gz.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys, os +sys.path.append(os.environ.get('TSTSRCDIR','.')) +import read_tests + +read_tests.read_concatenated('gz') diff --git a/common/stream/Tests/read_concatenated_lz4.py b/common/stream/Tests/read_concatenated_lz4.py new file mode 100644 --- /dev/null +++ b/common/stream/Tests/read_concatenated_lz4.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys, os +sys.path.append(os.environ.get('TSTSRCDIR','.')) +import read_tests + +read_tests.read_concatenated('lz4') diff --git a/common/stream/Tests/read_concatenated_xz.py b/common/stream/Tests/read_concatenated_xz.py new file mode 100644 --- /dev/null +++ b/common/stream/Tests/read_concatenated_xz.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys, os +sys.path.append(os.environ.get('TSTSRCDIR','.')) +import read_tests + +read_tests.read_concatenated('xz') diff --git a/common/stream/Tests/read_tests.py b/common/stream/Tests/read_tests.py --- a/common/stream/Tests/read_tests.py +++ b/common/stream/Tests/read_tests.py @@ -115,6 +115,27 @@ def all_tests(filename_filter): return failures +def read_concatenated(ext): + tf1 = TestFile("concatenated1", ext) + tf1.write(b"hi") + compressed_content = open(tf1.path(), "rb").read() + + tf2 = TestFile("concatenated2", ext) + tf2.write_raw(compressed_content + compressed_content) + + cmd = ['streamcat', 'read', tf2.path(), "rstream"] + results = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if results.returncode != 0 or results.stderr: + raise SystemExit(f"Command {cmd} returned with exit code {results.returncode}:\n{results.stderr or ''}") + + output = results.stdout + + expected = b'hihi' + if output != expected: + raise SystemExit(f"Expected {expected!r}, got {output!r}") + + if __name__ == "__main__": # generate test data for manual testing if len(sys.argv) == 1: diff --git a/common/stream/Tests/testdata.py b/common/stream/Tests/testdata.py --- a/common/stream/Tests/testdata.py +++ b/common/stream/Tests/testdata.py @@ -218,6 +218,13 @@ class TestFile: f.write(content) return filename + + def write_raw(self, content): + filename = self.path() + f = open(filename, 'wb') + f.write(content) + return filename + def read(self): filename = self.path() if not self.compression: diff --git a/common/stream/bz2_stream.c b/common/stream/bz2_stream.c --- a/common/stream/bz2_stream.c +++ b/common/stream/bz2_stream.c @@ -20,6 +20,7 @@ struct inner_state { bz_stream strm; int (*work)(bz_stream *strm, int flush); int (*end)(bz_stream *strm); + void (*reset)(inner_state_t *inner_state); bool eof_reached; char buf[64*1024]; }; @@ -108,6 +109,11 @@ work(inner_state_t *inner_state, pump_ac /* flushing and finishing is not yet done */ return PUMP_OK; case BZ_STREAM_END: + if (action == PUMP_NO_FLUSH && inner_state->reset != NULL) { + // attempt to read concatenated additional bz2 stream + inner_state->reset(inner_state); + return PUMP_OK; + } inner_state->eof_reached = true; return PUMP_END; default: @@ -122,6 +128,13 @@ finalizer(inner_state_t *inner_state) free(inner_state); } +static const char* +bz2_get_error(inner_state_t *inner_state) +{ + int dummy; + return BZ2_bzerror(&inner_state->strm, &dummy); +} + static int BZ2_bzDecompress_wrapper(bz_stream *strm, int a) { @@ -129,6 +142,17 @@ BZ2_bzDecompress_wrapper(bz_stream *strm return BZ2_bzDecompress(strm); } +static void +bz2_decompress_reset(inner_state_t *inner_state) +{ + pump_buffer src = get_src_win(inner_state); + pump_buffer dst = get_dst_win(inner_state); + BZ2_bzDecompressEnd(&inner_state->strm); + BZ2_bzDecompressInit(&inner_state->strm, 0, 0); + set_src_win(inner_state, src); + set_dst_win(inner_state, dst); +} + stream * bz2_stream(stream *inner, int level) { @@ -148,12 +172,14 @@ bz2_stream(stream *inner, int level) state->set_dst_win = set_dst_win; state->get_buffer = get_buffer; state->worker = work; + state->get_error = bz2_get_error; state->finalizer = finalizer; int ret; if (inner->readonly) { bz->work = BZ2_bzDecompress_wrapper; bz->end = BZ2_bzDecompressEnd; + bz->reset = bz2_decompress_reset; _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
