Changeset: 19e7e24ec8fd for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=19e7e24ec8fd
Modified Files:
clients/examples/C/streamcat.c
common/stream/Tests/write_bz2.stable.out
common/stream/Tests/write_gz.stable.out
common/stream/Tests/write_lz4.stable.out
common/stream/Tests/write_tests.py
common/stream/Tests/write_uncompressed.stable.out
common/stream/Tests/write_xz.stable.out
Branch: makelibstreamgreatagain
Log Message:
Add tests for flushing a stream
diffs (204 lines):
diff --git a/clients/examples/C/streamcat.c b/clients/examples/C/streamcat.c
--- a/clients/examples/C/streamcat.c
+++ b/clients/examples/C/streamcat.c
@@ -30,6 +30,8 @@ const char *USAGE =
"With W_OPENER:\n"
" - wstream stream = open_wstream(filename)\n"
" - wastream stream = open_wastream(filename)\n"
+ " - blocksize:N Copy in blocks of this size\n"
+ " - flush:{data,all} Flush after each block\n"
"With R_WRAPPER:\n"
" - iconv:enc stream = iconv_rstream(stream, enc)\n"
" - blocksize:N Copy in blocks of this size\n"
@@ -61,7 +63,7 @@ static stream *wrapper_autocompression(s
static void copy_stream_to_file(stream *in, FILE *out, size_t bufsize);
-static void copy_file_to_stream(FILE *in, stream *out, size_t bufsize);
+static void copy_file_to_stream(FILE *in, stream *out, size_t bufsize, bool
do_flush, mnstr_flush_level flush_level);
_Noreturn static void croak(int status, const char *msg, ...)
__attribute__((__format__(__printf__, 2, 3)));
@@ -215,6 +217,8 @@ int cmd_write(char *argv[])
char *opener_name = NULL;
opener_fun opener;
size_t bufsize = 1024;
+ bool do_flush = false;
+ mnstr_flush_level flush_level = MNSTR_FLUSH_DATA;
FILE *in = NULL;
stream *s = NULL;
@@ -281,6 +285,16 @@ int cmd_write(char *argv[])
if (*end != '\0' || size <= 0)
croak(1, "invalid blocksize: %ld", size);
bufsize = size;
+ } else if (strcmp(wrapper_name, "flush") == 0) {
+ if (parms == NULL)
+ parms = "";
+ if (strcmp(parms, "data") == 0)
+ flush_level = MNSTR_FLUSH_DATA;
+ else if (strcmp(parms, "all") == 0)
+ flush_level = MNSTR_FLUSH_ALL;
+ else
+ croak(1, "flush wrapper parameter is either
\"data\" or \"all\"");
+ do_flush = true;
} else {
croak(1, "Unknown wrapper: %s", wrapper_name);
}
@@ -297,7 +311,7 @@ int cmd_write(char *argv[])
#endif
}
- copy_file_to_stream(in, s, bufsize);
+ copy_file_to_stream(in, s, bufsize, do_flush, flush_level);
mnstr_close(s);
return 0;
@@ -344,7 +358,7 @@ static void copy_stream_to_file(stream *
}
-static void copy_file_to_stream(FILE *in, stream *out, size_t bufsize)
+static void copy_file_to_stream(FILE *in, stream *out, size_t bufsize, bool
do_flush, mnstr_flush_level flush_level)
{
char *buffer;
size_t nread;
@@ -370,6 +384,9 @@ static void copy_file_to_stream(FILE *in
(unsigned long)nwritten, (unsigned long)nread,
total + (int64_t)nwritten, mnstr_error(out));
total += (int64_t)nwritten;
+ if (do_flush)
+ if (mnstr_flush(out, flush_level) != 0)
+ croak(2, "Flush failed after %" PRId64 " bytes:
%s", total, mnstr_error(out));
}
free(buffer);
@@ -532,7 +549,7 @@ int cmd_bstream(char *argv[])
FILE *f = fopen(filename, "r");
if (!f)
croak(2, "could not open '%s': %s", filename,
strerror(errno));
- copy_file_to_stream(f, bs, 42);
+ copy_file_to_stream(f, bs, 42, false, 0);
fclose(f);
mnstr_flush(bs, MNSTR_FLUSH_DATA);
}
diff --git a/common/stream/Tests/write_bz2.stable.out
b/common/stream/Tests/write_bz2.stable.out
--- a/common/stream/Tests/write_bz2.stable.out
+++ b/common/stream/Tests/write_bz2.stable.out
@@ -126,7 +126,13 @@ Test: write ['wstream', 'blocksize:10000
Test: write ['wastream', 'blocksize:1000000'] sherlock.txt.bz2
OK
-# 10:20:11 >
-# 10:20:11 > "Done."
-# 10:20:11 >
+Test: write ['wstream', 'blocksize:1024', 'flush:data'] flushdata.txt.bz2
+ OK
+Test: write ['wstream', 'blocksize:1024', 'flush:all'] flushall.txt.bz2
+ OK
+
+# 14:39:39 >
+# 14:39:39 > "Done."
+# 14:39:39 >
+
diff --git a/common/stream/Tests/write_gz.stable.out
b/common/stream/Tests/write_gz.stable.out
--- a/common/stream/Tests/write_gz.stable.out
+++ b/common/stream/Tests/write_gz.stable.out
@@ -126,7 +126,13 @@ Test: write ['wstream', 'blocksize:10000
Test: write ['wastream', 'blocksize:1000000'] sherlock.txt.gz
OK
-# 10:20:12 >
-# 10:20:12 > "Done."
-# 10:20:12 >
+Test: write ['wstream', 'blocksize:1024', 'flush:data'] flushdata.txt.gz
+ OK
+Test: write ['wstream', 'blocksize:1024', 'flush:all'] flushall.txt.gz
+ OK
+
+# 14:39:40 >
+# 14:39:40 > "Done."
+# 14:39:40 >
+
diff --git a/common/stream/Tests/write_lz4.stable.out
b/common/stream/Tests/write_lz4.stable.out
--- a/common/stream/Tests/write_lz4.stable.out
+++ b/common/stream/Tests/write_lz4.stable.out
@@ -126,7 +126,13 @@ Test: write ['wstream', 'blocksize:10000
Test: write ['wastream', 'blocksize:1000000'] sherlock.txt.lz4
OK
-# 09:27:04 >
-# 09:27:04 > "Done."
-# 09:27:04 >
+Test: write ['wstream', 'blocksize:1024', 'flush:data'] flushdata.txt.lz4
+ OK
+Test: write ['wstream', 'blocksize:1024', 'flush:all'] flushall.txt.lz4
+ OK
+
+# 14:39:40 >
+# 14:39:40 > "Done."
+# 14:39:40 >
+
diff --git a/common/stream/Tests/write_tests.py
b/common/stream/Tests/write_tests.py
--- a/common/stream/Tests/write_tests.py
+++ b/common/stream/Tests/write_tests.py
@@ -92,6 +92,7 @@ def gen_docs():
def gen_tests():
+ sherlock = Doc(testdata.SHERLOCK)
for compr in testdata.COMPRESSIONS:
for name, doc in gen_docs():
yield TestCase(name, doc, compr, "wstream", doc)
@@ -103,6 +104,8 @@ def gen_tests():
yield TestCase(name, doc, compr, ["wastream", "blocksize:2"],
doc.to_platform())
yield TestCase(name, doc, compr, ["wstream", "blocksize:1000000"],
doc)
yield TestCase(name, doc, compr, ["wastream",
"blocksize:1000000"], doc.to_platform())
+ for level in ['data', 'all']:
+ yield TestCase(f'flush{level}.txt', sherlock, compr, ["wstream",
"blocksize:1024", f"flush:{level}"], sherlock)
def all_tests(filename_filter):
diff --git a/common/stream/Tests/write_uncompressed.stable.out
b/common/stream/Tests/write_uncompressed.stable.out
--- a/common/stream/Tests/write_uncompressed.stable.out
+++ b/common/stream/Tests/write_uncompressed.stable.out
@@ -126,7 +126,13 @@ Test: write ['wstream', 'blocksize:10000
Test: write ['wastream', 'blocksize:1000000'] sherlock.txt
OK
-# 10:20:11 >
-# 10:20:11 > "Done."
-# 10:20:11 >
+Test: write ['wstream', 'blocksize:1024', 'flush:data'] flushdata.txt
+ OK
+Test: write ['wstream', 'blocksize:1024', 'flush:all'] flushall.txt
+ OK
+
+# 14:39:38 >
+# 14:39:38 > "Done."
+# 14:39:38 >
+
diff --git a/common/stream/Tests/write_xz.stable.out
b/common/stream/Tests/write_xz.stable.out
--- a/common/stream/Tests/write_xz.stable.out
+++ b/common/stream/Tests/write_xz.stable.out
@@ -126,7 +126,13 @@ Test: write ['wstream', 'blocksize:10000
Test: write ['wastream', 'blocksize:1000000'] sherlock.txt.xz
OK
-# 09:27:05 >
-# 09:27:05 > "Done."
-# 09:27:05 >
+Test: write ['wstream', 'blocksize:1024', 'flush:data'] flushdata.txt.xz
+ OK
+Test: write ['wstream', 'blocksize:1024', 'flush:all'] flushall.txt.xz
+ OK
+
+# 14:39:41 >
+# 14:39:41 > "Done."
+# 14:39:41 >
+
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
