Changeset: 839bd7161d60 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=839bd7161d60
Modified Files:
sql/backends/monet5/Tests/pyloader07.sql
sql/backends/monet5/Tests/pyloader07.stable.out
sql/backends/monet5/Tests/pyloader07.stable.out.Windows
sql/backends/monet5/UDF/pyapi/convert_loops.h
sql/backends/monet5/UDF/pyapi/emit.c
Branch: Jul2017
Log Message:
Properly support dates, times and timestamps in Python loader functions.
diffs (247 lines):
diff --git a/sql/backends/monet5/Tests/pyloader07.sql
b/sql/backends/monet5/Tests/pyloader07.sql
--- a/sql/backends/monet5/Tests/pyloader07.sql
+++ b/sql/backends/monet5/Tests/pyloader07.sql
@@ -9,3 +9,21 @@ SELECT * FROM pyloader07table;
DROP TABLE pyloader07table;
DROP LOADER pyloader07;
ROLLBACK;
+
+START TRANSACTION;
+
+CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+
+
+CREATE LOADER pyloader07() LANGUAGE PYTHON {
+ _emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20
00:02:30'});
+ _emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20
00:02:30']});
+};
+
+COPY LOADER INTO tstamp FROM pyloader07();
+
+SELECT * FROM tstamp;
+DROP TABLE tstamp;
+DROP LOADER pyloader07;
+
+ROLLBACK;
diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out
b/sql/backends/monet5/Tests/pyloader07.stable.out
--- a/sql/backends/monet5/Tests/pyloader07.stable.out
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out
@@ -75,6 +75,24 @@ Ready.
#DROP TABLE pyloader07table;
#DROP LOADER pyloader07;
#ROLLBACK;
+#START TRANSACTION;
+#CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+#CREATE LOADER pyloader07() LANGUAGE PYTHON {
+# _emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20
00:02:30'});
+# _emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20
00:02:30']});
+#};
+#COPY LOADER INTO tstamp FROM pyloader07();
+[ 2 ]
+#SELECT * FROM tstamp;
+% sys.tstamp, sys.tstamp, sys.tstamp # table_name
+% d, s, t # name
+% date, time, timestamp # type
+% 10, 8, 26 # length
+[ 2014-05-20, 00:02:30, 2014-05-20 00:02:30.000000 ]
+[ 2014-05-20, 00:02:30, 2014-05-20 00:02:30.000000 ]
+#DROP TABLE tstamp;
+#DROP LOADER pyloader07;
+#ROLLBACK;
# 15:06:46 >
# 15:06:46 > "Done."
diff --git a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
--- a/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
+++ b/sql/backends/monet5/Tests/pyloader07.stable.out.Windows
@@ -75,6 +75,24 @@ Ready.
#DROP TABLE pyloader07table;
#DROP LOADER pyloader07;
#ROLLBACK;
+#START TRANSACTION;
+#CREATE TABLE tstamp(d DATE, s TIME, t TIMESTAMP);
+#CREATE LOADER pyloader07() LANGUAGE PYTHON {
+# _emit.emit({'d': '2014-05-20', 's': '00:02:30', 't': '2014-05-20
00:02:30'});
+# _emit.emit({'d': ['2014-05-20'], 's': ['00:02:30'], 't': ['2014-05-20
00:02:30']});
+#};
+#COPY LOADER INTO tstamp FROM pyloader07();
+[ 2 ]
+#SELECT * FROM tstamp;
+% sys.tstamp, sys.tstamp, sys.tstamp # table_name
+% d, s, t # name
+% date, time, timestamp # type
+% 10, 8, 26 # length
+[ 2014-05-20, 00:02:30, 2014-05-20 00:02:30.000000 ]
+[ 2014-05-20, 00:02:30, 2014-05-20 00:02:30.000000 ]
+#DROP TABLE tstamp;
+#DROP LOADER pyloader07;
+#ROLLBACK;
# 15:06:46 >
# 15:06:46 > "Done."
diff --git a/sql/backends/monet5/UDF/pyapi/convert_loops.h
b/sql/backends/monet5/UDF/pyapi/convert_loops.h
--- a/sql/backends/monet5/UDF/pyapi/convert_loops.h
+++ b/sql/backends/monet5/UDF/pyapi/convert_loops.h
@@ -282,6 +282,24 @@
}
\
}
+
+static gdk_return
+convert_and_append(BAT* b, const char* text, bit force) {
+ if (b->ttype == TYPE_str) {
+ return BUNappend(b, text, force);
+ } else if (text == str_nil) {
+ return BUNappend(b, BATatoms[b->ttype].atomNull, force);
+ } else {
+ void* element = NULL;
+ int len = 0;
+
+ BATatoms[b->ttype].atomFromStr(text, &len, &element);
+ gdk_return ret = BUNappend(b, element, force);
+ GDKfree(element);
+ return ret;
+ }
+}
+
// This #define is for converting a numeric numpy array into a string BAT.
// 'conv' is a function that turns a numeric value of type 'mtpe' to a char*
// array.
@@ -291,7 +309,7 @@
snprintf(utf8_string, utf8string_minlength, fmt,
\
*((mtpe *)&data[(index_offset *
ret->count + iu) * \
ret->memory_size])); \
- if (BUNappend(bat, utf8_string, FALSE) != GDK_SUCCEED)
{ \
+ if (convert_and_append(bat, utf8_string, FALSE) !=
GDK_SUCCEED) { \
msg =
\
createException(MAL, "pyapi.eval",
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -301,7 +319,7 @@
for (iu = 0; iu < ret->count; iu++) {
\
if (mask[index_offset * ret->count + iu] == TRUE) {
\
bat->tnil = 1;
\
- if (BUNappend(bat, str_nil, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(bat, str_nil, FALSE) !=
GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -310,7 +328,7 @@
snprintf(utf8_string, utf8string_minlength,
fmt, \
*((mtpe *)&data[(index_offset
* ret->count + iu) * \
ret->memory_size])); \
- if (BUNappend(bat, utf8_string, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(bat, utf8_string, FALSE)
!= GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -434,7 +452,7 @@
if (mask != NULL &&
\
(mask[index_offset * ret->count + iu])
== TRUE) { \
b->tnil = 1;
\
- if (BUNappend(b, str_nil, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(b, str_nil,
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -450,7 +468,7 @@
"object.\n"); \
goto wrapup;
\
}
\
- if (BUNappend(b, utf8_string, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(b, utf8_string,
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -463,7 +481,7 @@
if (mask != NULL &&
\
(mask[index_offset * ret->count + iu])
== TRUE) { \
b->tnil = 1;
\
- if (BUNappend(b, str_nil, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(b, str_nil,
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
@@ -474,7 +492,7 @@
(const Py_UNICODE
\
*)(&data[(index_offset
* ret->count + iu) * \
ret->memory_size])); \
- if (BUNappend(b, utf8_string, FALSE) !=
GDK_SUCCEED) { \
+ if (convert_and_append(b, utf8_string,
FALSE) != GDK_SUCCEED) { \
msg = createException(MAL,
"pyapi.eval", \
"BUNappend failed.\n"); \
goto wrapup;
\
diff --git a/sql/backends/monet5/UDF/pyapi/emit.c
b/sql/backends/monet5/UDF/pyapi/emit.c
--- a/sql/backends/monet5/UDF/pyapi/emit.c
+++ b/sql/backends/monet5/UDF/pyapi/emit.c
@@ -12,6 +12,7 @@
#include "convert_loops.h"
#include "type_conversion.h"
#include "gdk_interprocess.h"
+#include "mtime.h"
#include "unicode.h"
@@ -251,7 +252,7 @@ PyObject *PyEmit_Emit(PyEmitObject *self
scalar_convert(hge);
break;
#endif
- case TYPE_str: {
+ default: {
str val = NULL;
gdk_return retval;
msg =
pyobject_to_str(&dictEntry, 42, &val);
@@ -259,18 +260,13 @@ PyObject *PyEmit_Emit(PyEmitObject *self
goto wrapup;
}
assert(val);
- retval =
BUNappend(self->cols[i].b, val, 0);
+ retval =
convert_and_append(self->cols[i].b, val, 0);
free(val);
if (retval != GDK_SUCCEED) {
msg =
GDKstrdup("BUNappend failed.");
goto wrapup;
}
- } break;
- default:
- PyErr_Format(PyExc_TypeError,
"Unsupported BAT Type %s",
-
BatType_Format(self->cols[i].b->ttype));
- error = true;
- goto wrapup;
+ }
}
} else {
bool *mask = NULL;
@@ -295,6 +291,7 @@ PyObject *PyEmit_Emit(PyEmitObject *self
mask = (bool *)ret->mask_data;
data = (char *)ret->array_data;
assert((size_t)el_count == (size_t)ret->count);
+
switch (self->cols[i].b->ttype) {
case TYPE_bit:
NP_INSERT_BAT(self->cols[i].b,
bit, self->nvals);
@@ -325,7 +322,7 @@ PyObject *PyEmit_Emit(PyEmitObject *self
NP_INSERT_BAT(self->cols[i].b,
hge, self->nvals);
break;
#endif
- case TYPE_str: {
+ default: {
char *utf8_string = NULL;
if (ret->result_type !=
NPY_OBJECT) {
utf8_string =
GDKzalloc(utf8string_minlength +
@@ -334,14 +331,8 @@ PyObject *PyEmit_Emit(PyEmitObject *self
ret->memory_size] = '\0';
}
NP_INSERT_STRING_BAT(self->cols[i].b);
- if (utf8_string)
- GDKfree(utf8_string);
- } break;
- default:
- PyErr_Format(PyExc_TypeError,
"Unsupported BAT Type %s",
-
BatType_Format(self->cols[i].b->ttype));
- error = true;
- goto wrapup;
+ GDKfree(utf8_string);
+ }
}
self->cols[i].b->tnonil = 1 -
self->cols[i].b->tnil;
}
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
