Changeset: 4c10ac6baff6 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4c10ac6baff6
Added Files:
common/stream/Tests/read_iconv.py
common/stream/Tests/read_iconv.stable.err
common/stream/Tests/read_iconv.stable.out
common/stream/Tests/write_iconv.py
common/stream/Tests/write_iconv.stable.err
common/stream/Tests/write_iconv.stable.out
Modified Files:
clients/examples/C/streamcat.c
common/stream/Tests/All
common/stream/Tests/testdata.py
Branch: makelibstreamgreatagain
Log Message:
Add tests for iconv stream
diffs (truncated from 307 to 300 lines):
diff --git a/clients/examples/C/streamcat.c b/clients/examples/C/streamcat.c
--- a/clients/examples/C/streamcat.c
+++ b/clients/examples/C/streamcat.c
@@ -10,14 +10,18 @@
const char *USAGE =
"Usage:\n"
- " streamcat read FILENAME R_OPENER\n"
- " streamcat write FILENAME W_OPENER\n"
+ " streamcat read FILENAME R_OPENER [R_WRAPPER..]\n"
+ " streamcat write FILENAME W_OPENER [W_WRAPPER..]\n"
"With R_OPENER:\n"
" - rstream stream = open_rstream(filename)\n"
" - rastream stream = open_rastream(filename)\n"
"With W_OPENER:\n"
" - wstream stream = open_wstream(filename)\n"
" - wastream stream = open_wastream(filename)\n"
+ "With R_WRAPPER:\n"
+ " - iconv:enc stream = iconv_rstream(stream, enc)\n"
+ "With W_WRAPPER:\n"
+ " - iconv:enc stream = iconv_wstream(stream, enc)\n"
;
@@ -31,6 +35,10 @@ static stream *opener_rastream(char *fil
static stream *opener_wstream(char *filename);
static stream *opener_wastream(char *filename);
+static stream *wrapper_read_iconv(stream *s, char *enc);
+
+static stream *wrapper_write_iconv(stream *s, char *enc);
+
static void copy_to_stdout(stream *in);
static void copy_from_stdin(stream *out);
@@ -98,6 +106,29 @@ int cmd_read(char *argv[])
croak(1, "Unknown opener '%s'", opener_name);
s = opener(filename);
+ if (s == NULL)
+ croak(2, "Opener %s did not return a stream", opener_name);
+
+ for (; *arg != NULL; arg++) {
+ char *wrapper_name = *arg;
+ char *parms = strchr(wrapper_name, ':');
+ stream *(*wrapper)(stream *s, char *parm) = NULL;
+
+ if (parms != NULL) {
+ *parms = '\0';
+ parms += 1;
+ }
+ if (strcmp(wrapper_name, "iconv") == 0) {
+ if (parms == NULL)
+ croak(1, "iconv wrapper needs a parameter");
+ wrapper = wrapper_read_iconv;
+ }
+ if (wrapper == NULL)
+ croak(1, "Unknown wrapper: %s", wrapper_name);
+ s = wrapper(s, parms);
+ if (s == NULL)
+ croak(2, "Wrapper %s did not return a stream",
wrapper_name);
+ }
copy_to_stdout(s);
mnstr_close(s);
@@ -130,6 +161,29 @@ int cmd_write(char *argv[])
croak(1, "Unknown opener '%s'", opener_name);
s = opener(filename);
+ if (s == NULL)
+ croak(2, "Opener %s did not return a stream", opener_name);
+
+ for (; *arg != NULL; arg++) {
+ char *wrapper_name = *arg;
+ char *parms = strchr(wrapper_name, ':');
+ stream *(*wrapper)(stream *s, char *parm) = NULL;
+
+ if (parms != NULL) {
+ *parms = '\0';
+ parms += 1;
+ }
+ if (strcmp(wrapper_name, "iconv") == 0) {
+ if (parms == NULL)
+ croak(1, "iconv wrapper needs a parameter");
+ wrapper = wrapper_write_iconv;
+ }
+ if (wrapper == NULL)
+ croak(1, "Unknown wrapper: %s", wrapper_name);
+ s = wrapper(s, parms);
+ if (s == NULL)
+ croak(2, "Wrapper %s did not return a stream",
wrapper_name);
+ }
copy_from_stdin(s);
mnstr_close(s);
@@ -238,3 +292,17 @@ opener_wastream(char *filename)
croak(2, "Error opening file '%s': %s", filename,
strerror(errno));
return s;
}
+
+
+static stream *
+wrapper_read_iconv(stream *s, char *enc)
+{
+ return iconv_rstream(s, enc, "wrapper_read_iconv");
+}
+
+
+static stream *
+wrapper_write_iconv(stream *s, char *enc)
+{
+ return iconv_wstream(s, enc, "wrapper_write_iconv");
+}
diff --git a/common/stream/Tests/All b/common/stream/Tests/All
--- a/common/stream/Tests/All
+++ b/common/stream/Tests/All
@@ -9,3 +9,6 @@ HAVE_LIBBZ2?write_bz2
HAVE_LIBZ?write_gz
HAVE_LIBLZMA?write_lz4
HAVE_LIBLZ4?write_xz
+
+read_iconv
+write_iconv
\ No newline at end of file
diff --git a/common/stream/Tests/read_iconv.py
b/common/stream/Tests/read_iconv.py
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/read_iconv.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+from testdata import Doc
+
+import os
+import subprocess
+import sys
+
+def run_streamcat(text, enc):
+ content = bytes(text, enc)
+ d = Doc(f'read_iconv_{enc}.txt', content)
+ filename = d.write_tmp()
+ cmd = ['streamcat', 'read', filename, 'rstream', f'iconv:{enc}']
+ print(f"Input with encoding '{enc}' is {repr(content)}")
+ #print(cmd)
+ proc = subprocess.run(cmd, stdout=subprocess.PIPE)
+ if proc.returncode != 0:
+ print(f"{cmd} exited with status {proc.returncode}",
file=sys.stderr)
+ sys.exit(1)
+ os.remove(filename)
+ print(f"Output is {repr(proc.stdout)}")
+ print()
+ return proc.stdout
+
+
+text = "MøNëTDB"
+run_streamcat(text, 'utf-8')
+run_streamcat(text, 'latin1')
diff --git a/common/stream/Tests/read_iconv.stable.err
b/common/stream/Tests/read_iconv.stable.err
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/read_iconv.stable.err
@@ -0,0 +1,12 @@
+stderr of test 'read_iconv` in directory 'common/stream` itself:
+
+
+# 15:01:38 >
+# 15:01:38 > "/usr/bin/python3" "read_iconv.py" "read_iconv"
+# 15:01:38 >
+
+
+# 15:01:38 >
+# 15:01:38 > "Done."
+# 15:01:38 >
+
diff --git a/common/stream/Tests/read_iconv.stable.out
b/common/stream/Tests/read_iconv.stable.out
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/read_iconv.stable.out
@@ -0,0 +1,18 @@
+stdout of test 'read_iconv` in directory 'common/stream` itself:
+
+
+# 15:01:38 >
+# 15:01:38 > "/usr/bin/python3" "read_iconv.py" "read_iconv"
+# 15:01:38 >
+
+Input with encoding 'utf-8' is b'M\xc3\xb8N\xc3\xabTDB'
+Output is b'M\xc3\xb8N\xc3\xabTDB'
+
+Input with encoding 'latin1' is b'M\xf8N\xebTDB'
+Output is b'M\xc3\xb8N\xc3\xabTDB'
+
+
+# 15:01:38 >
+# 15:01:38 > "Done."
+# 15:01:38 >
+
diff --git a/common/stream/Tests/testdata.py b/common/stream/Tests/testdata.py
--- a/common/stream/Tests/testdata.py
+++ b/common/stream/Tests/testdata.py
@@ -93,11 +93,7 @@ class Doc:
def pick_tmp_name(self, dir=None):
prefix = "_streamtest_"
suffix = "_" + self.name
- dir = dir or TMPDIR or None
- h, p = tempfile.mkstemp(suffix, prefix, dir)
- os.close(h)
- os.remove(p)
- return p
+ return pick_tmp_name(suffix, prefix, dir)
def write_tmp(self, dir=None):
p = self.pick_tmp_name(dir)
@@ -159,8 +155,17 @@ class Doc:
return None
+def pick_tmp_name(suffix, prefix, dir=None):
+ dir = dir or TMPDIR or None
+ h, p = tempfile.mkstemp(suffix, prefix, dir)
+ os.close(h)
+ os.remove(p)
+ return p
+
+
+
# test code
if __name__ == "__main__":
- d = Doc("banana")
+ d = Doc("banana", "banana")
p = d.write_tmp()
print(f"path to doc {d.name} is {p}")
diff --git a/common/stream/Tests/write_iconv.py
b/common/stream/Tests/write_iconv.py
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/write_iconv.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+from testdata import Doc, pick_tmp_name
+
+import os
+import subprocess
+import sys
+import tempfile
+
+def run_streamcat(text, enc):
+ input = bytes(text, 'utf-8')
+ filename = pick_tmp_name('_streamtest_write_iconv', '.txt')
+
+ cmd = ['streamcat', 'write', filename, 'wstream', f'iconv:{enc}']
+
+ print(f"Input UTF-8 encoded is {repr(input)}")
+ #print(cmd)
+ proc = subprocess.run(cmd, input=input)
+ if proc.returncode != 0:
+ print(f"{cmd} exited with status {proc.returncode}",
file=sys.stderr)
+ sys.exit(1)
+ output = open(filename, 'rb').read()
+ os.remove(filename)
+ print(f"Output is {repr(output)}")
+ print()
+
+
+text = "MøNëTDB"
+run_streamcat(text, 'utf-8')
+run_streamcat(text, 'latin1')
diff --git a/common/stream/Tests/write_iconv.stable.err
b/common/stream/Tests/write_iconv.stable.err
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/write_iconv.stable.err
@@ -0,0 +1,12 @@
+stderr of test 'write_iconv` in directory 'common/stream` itself:
+
+
+# 15:02:03 >
+# 15:02:03 > "/usr/bin/python3" "write_iconv.py" "write_iconv"
+# 15:02:03 >
+
+
+# 15:02:03 >
+# 15:02:03 > "Done."
+# 15:02:03 >
+
diff --git a/common/stream/Tests/write_iconv.stable.out
b/common/stream/Tests/write_iconv.stable.out
new file mode 100644
--- /dev/null
+++ b/common/stream/Tests/write_iconv.stable.out
@@ -0,0 +1,18 @@
+stdout of test 'write_iconv` in directory 'common/stream` itself:
+
+
+# 15:02:03 >
+# 15:02:03 > "/usr/bin/python3" "write_iconv.py" "write_iconv"
+# 15:02:03 >
+
+Input UTF-8 encoded is b'M\xc3\xb8N\xc3\xabTDB'
+Output is b'M\xc3\xb8N\xc3\xabTDB'
+
+Input UTF-8 encoded is b'M\xc3\xb8N\xc3\xabTDB'
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
