Changeset: 90f85fc4830a for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=90f85fc4830a
Added Files:
monetdb5/extras/pyapi/emit.c
monetdb5/extras/pyapi/emit.h
monetdb5/extras/pyapi/pyloader.c
sql/backends/monet5/Tests/pyapi30.stable.err
sql/backends/monet5/Tests/pyapi30.stable.out
sql/backends/monet5/Tests/pyapi31.sql
sql/backends/monet5/Tests/pyapi31.stable.err
sql/backends/monet5/Tests/pyapi31.stable.out
Modified Files:
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/formatinput.c
monetdb5/extras/pyapi/formatinput.h
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.h
monetdb5/extras/pyapi/pyapi.mal
monetdb5/extras/rapi/rapi.c
sql/backends/monet5/Tests/All
sql/backends/monet5/Tests/pyapi30.sql
sql/backends/monet5/sql_gencode.c
sql/include/sql_catalog.h
sql/server/rel_psm.c
sql/server/rel_select.c
sql/server/rel_updates.c
sql/server/sql_mvc.h
sql/server/sql_parser.y
Branch: pythonloader
Log Message:
second stage COPY INTO sometable FROM LOADER somepythonfunction(); basic
functionality is there
diffs (truncated from 1512 to 300 lines):
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -24,7 +24,7 @@ MTSAFE
lib__pyapi = {
MODULE
DIR = libdir/monetdb5
- SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c
connection.h unspecified_evil.h
+ SOURCES = pyapi.c pyapi.h unicode.c unicode.h pytypes.c pytypes.h
type_conversion.c type_conversion.h formatinput.c formatinput.h connection.c
connection.h unspecified_evil.h pyloader.c emit.h emit.c
XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
../../../gdk/libbat \
diff --git a/monetdb5/extras/pyapi/emit.c b/monetdb5/extras/pyapi/emit.c
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/pyapi/emit.c
@@ -0,0 +1,195 @@
+
+#include "emit.h"
+#include "type_conversion.h"
+#include "interprocess.h"
+
+#if PY_MAJOR_VERSION >= 3
+#define IS_PY3K
+#define PyString_CheckExact PyUnicode_CheckExact
+#define PyString_FromString PyUnicode_FromString
+#endif
+
+#define scalar_convert(tpe) {\
+ tpe val = (tpe) tpe##_nil; msg = pyobject_to_##tpe(&dictEntry, 42,
&val); \
+ BUNappend(self->cols[i].b, &val, 0); \
+ if (msg != MAL_SUCCEED) { \
+ PyErr_SetString(PyExc_TypeError, "conversion failed"); /* TODO:
better error message */ \
+ return NULL; \
+ }}
+
+static PyObject *
+_emit_emit(Py_EmitObject *self, PyObject *args) {
+ size_t i, ai;
+ ssize_t el_count = -1;
+ str msg = MAL_SUCCEED;
+ (void) self;
+ if (!PyDict_Check(args)) {
+ // complain
+ PyErr_SetString(PyExc_TypeError, "need dict");
+ return NULL;
+ }
+
+ for (i = 0; i < self->ncols; i++) {
+ PyObject *dictEntry = PyDict_GetItemString(args,
self->cols[i].name);
+ ssize_t this_size = 1;
+ if (dictEntry) {
+ if (!PyType_IsPyScalar(dictEntry)) {
+ this_size = Py_SIZE(dictEntry);
+ }
+ if (el_count < 0) el_count = this_size;
+ else {
+ if (el_count != this_size) {
+ PyErr_SetString(PyExc_TypeError, "need
same length values");
+ // todo: better error message!
+ return NULL;
+ }
+ }
+ }
+ }
+ if (el_count < 1) {
+ PyErr_SetString(PyExc_TypeError, "need at least some values");
+ // todo: better error message!
+ return NULL;
+ }
+
+ // TODO: check for dict entries not matched by any column and complain
if present
+
+
+ for (i = 0; i < self->ncols; i++) {
+ PyObject *dictEntry = PyDict_GetItemString(args,
self->cols[i].name);
+ if (dictEntry) {
+ if (PyType_IsPyScalar(dictEntry)) {
+ switch (self->cols[i].b->T->type)
+ {
+ case TYPE_bit:
+ scalar_convert(bit);
+ break;
+ case TYPE_bte:
+ scalar_convert(bte);
+ break;
+ case TYPE_sht:
+ scalar_convert(sht);
+ break;
+ case TYPE_int:
+ scalar_convert(int);
+ break;
+ case TYPE_oid:
+ scalar_convert(oid);
+ break;
+ case TYPE_lng:
+ scalar_convert(lng);
+ break;
+ case TYPE_flt:
+ scalar_convert(flt);
+ break;
+ case TYPE_dbl:
+ scalar_convert(dbl);
+ break;
+ #ifdef HAVE_HGE
+ case TYPE_hge:
+ scalar_convert(hge);
+ break;
+ #endif
+ case TYPE_str:
+ // FIXME scalar_convert(str);
+ break;
+ default:
+ break;
+ // complain
+ }
+ } else {
+ // TODO: handle dicts with array values
+ }
+ } else {
+ for (ai = 0; ai < (size_t) el_count; ai++) {
+ BUNappend(self->cols[i].b,
ATOMnil(self->cols[i].b->T->type), 0);
+ }
+ }
+ }
+ self->nvals += el_count;
+ Py_RETURN_NONE;
+}
+
+
+static PyMethodDef _emitObject_methods[] = {
+ {"emit", (PyCFunction)_emit_emit, METH_O,"emit(dictionary) -> returns
parsed values for table insertion"},
+ {NULL,NULL,0,NULL} /* Sentinel */
+};
+
+PyTypeObject Py_EmitType = {
+ PyObject_HEAD_INIT(NULL)
+ 0,
+ "monetdb._emit",
+ sizeof(Py_EmitObject),
+ 0,
+ 0, /* tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ 0, /* tp_compare */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ (hashfunc)PyObject_HashNotImplemented, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ 0, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT, /* tp_flags */
+ "Value Emitter", /* tp_doc */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
+ _emitObject_methods, /* tp_methods */
+ 0, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ PyType_GenericAlloc, /* tp_alloc */
+ PyType_GenericNew, /* tp_new */
+ PyObject_Del, /* tp_free */
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+#ifdef IS_PY3K
+ ,0
+#endif
+};
+
+
+
+PyObject *Py_Emit_Create(EmitCol *cols, size_t ncols)
+{
+ register Py_EmitObject *op;
+
+ op = (Py_EmitObject *)PyObject_MALLOC(sizeof(Py_EmitObject));
+ if (op == NULL)
+ return PyErr_NoMemory();
+ PyObject_Init((PyObject*)op, &Py_EmitType);
+ op->cols = cols;
+ op->ncols = ncols;
+ op->nvals = 0;
+ return (PyObject*) op;
+}
+
+str _emit_init(void)
+{
+ _import_array();
+ if (PyType_Ready(&Py_EmitType) < 0)
+ return createException(MAL, "pyapi.eval", "Failed to initialize emit
type.");
+ return MAL_SUCCEED;
+}
diff --git a/monetdb5/extras/pyapi/emit.h b/monetdb5/extras/pyapi/emit.h
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/pyapi/emit.h
@@ -0,0 +1,39 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 2008-2015 MonetDB B.V.
+ */
+
+/*
+ * M. Raasveldt
+ *
+ */
+
+#ifndef _LOADER_EMIT_
+#define _LOADER_EMIT_
+
+#include "pytypes.h"
+
+
+typedef struct {
+ BAT *b;
+ char* name;
+} EmitCol;
+
+typedef struct {
+ PyObject_HEAD
+ // some more stuff
+ EmitCol *cols;
+ size_t ncols;
+ size_t nvals;
+} Py_EmitObject;
+
+extern PyTypeObject Py_EmitType;
+
+PyObject *Py_Emit_Create(EmitCol *cols, size_t ncols);
+
+str _emit_init(void);
+
+#endif /* _LOADER_EMIT_ */
diff --git a/monetdb5/extras/pyapi/formatinput.c
b/monetdb5/extras/pyapi/formatinput.c
--- a/monetdb5/extras/pyapi/formatinput.c
+++ b/monetdb5/extras/pyapi/formatinput.c
@@ -7,9 +7,6 @@
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#endif
-const size_t additional_argcount = 3;
-const char * additional_args[] = {"_columns", "_column_types", "_conn"};
-
#if PY_MAJOR_VERSION >= 3
#define IS_PY3K
#endif
@@ -61,7 +58,7 @@ PyObject *PyCodeObject_ParseString(char
return code_object;
}
-char* FormatCode(char* code, char **args, size_t argcount, size_t tabwidth,
PyObject **code_object, char **msg)
+char* FormatCode(char* code, char **args, size_t argcount, size_t tabwidth,
PyObject **code_object, char **msg, char** additional_args, size_t
additional_argcount)
{
// Format the python code by fixing the indentation levels
// We do two passes, first we get the length of the resulting formatted
code and then we actually create the resulting code
diff --git a/monetdb5/extras/pyapi/formatinput.h
b/monetdb5/extras/pyapi/formatinput.h
--- a/monetdb5/extras/pyapi/formatinput.h
+++ b/monetdb5/extras/pyapi/formatinput.h
@@ -20,7 +20,7 @@
extern PyObject *marshal_loads;
-char* FormatCode(char* code, char **args, size_t argcount, size_t tabwidth,
PyObject **code_object, char **return_message);
+char* FormatCode(char* code, char **args, size_t argcount, size_t tabwidth,
PyObject **code_object, char **msg, char** additional_args, size_t
additional_argcount);
void _formatinput_init(void);
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -36,7 +36,6 @@
#endif
-const char* pyapi_enableflag = "embedded_py";
const char* verbose_enableflag = "enable_pyverbose";
const char* warning_enableflag = "enable_pywarnings";
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
