MonetDB: pyapi - Python API: Full circle and first test output
Changeset: fa5bedf47134 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fa5bedf47134
Modified Files:
monetdb5/extras/pyapi/Tests/pyapi00.malC
monetdb5/extras/pyapi/pyapi.c
Branch: pyapi
Log Message:
Python API: Full circle and first test output
diffs (87 lines):
diff --git a/monetdb5/extras/pyapi/Tests/pyapi00.malC
b/monetdb5/extras/pyapi/Tests/pyapi00.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi00.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi00.malC
@@ -1,18 +1,18 @@
b:= bat.new(:oid,:int);
-bat.append(b,1804289383);
-bat.append(b,846930886);
-bat.append(b,1681692777);
-bat.append(b,1714636915);
-bat.append(b,1957747793);
-bat.append(b,424238335);
-bat.append(b,719885386);
-bat.append(b,1649760492);
-bat.append(b,596516649);
-bat.append(b,1189641421);
+bat.append(b, 42);
+bat.append(b, 43);
+bat.append(b, 44);
+bat.append(b, 45);
+bat.append(b, 46);
+bat.append(b, 47);
+bat.append(b, 48);
+bat.append(b, 49);
+bat.append(b, 50);
+bat.append(b, 51);
io.print(b);
-r:bat[:oid,:int] := pyapi.eval(nil:ptr,r=[e**2 for e in arg1]\nreturn
([numpy.asarray(r)]),b);
+r:bat[:oid,:int] := pyapi.eval(nil:ptr, r=[e+1 for e in arg1]\nreturn ([r]),
b);
io.print(r);
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -250,7 +250,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
#ifdef _PYAPI_DEBUG_
- printf(# Actual Python function definition: %s\n,pycall);
+ printf(Actual Python function definition: \n%s\n, pycall);
#endif
{
@@ -278,6 +278,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
goto wrapup; // shudder
}
+ // TODO: handle the case where only a single array is returned
(nice to have)
if (!pResult || !PyList_Check(pResult) || PyList_Size(pResult)
!= pci-retc) {
msg = createException(MAL, pyapi.eval, Invalid
result object. Need list of size %d containing numpy arrays, pci-retc);
goto wrapup;
@@ -293,28 +294,24 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
switch (bat_type) {
case TYPE_int: {
- int *p;
BUN j;
// this only copies if it has to
PyArrayObject* pCol = (PyArrayObject*)
PyArray_FromAny(pColO,
PyArray_DescrFromType(NPY_INT32), 1, 1,
NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, NULL);
- //size_t cnt = pCol-dimensions[0];
- size_t cnt = 5;
- // TODO: get actual length from array
+ size_t cnt = PyArray_DIMS(pCol)[0];
// TODO null rewriting, we are guaranteed to be able to
write to this
- // TODO: only accepted masked array as output?
- // TODO check whether the length of our output
/* We would like to simply pass over the BAT from numpy,
* but cannot due to malloc/free incompatibility */
+
b = BATnew(TYPE_void, TYPE_int, cnt, TRANSIENT);
BATseqbase(b, 0); b-T-nil = 0; b-T-nonil = 1;
b-tkey = 0;
b-tsorted = 0; b-trevsorted = 0;
- p = (int*) Tloc(b, BUNfirst(b));
\
- for( j =0; j cnt; j++, p++){
- *p = (int) PyArray_GETPTR1(pCol, j);
+ for( j =0; j cnt; j++){
+ ((int*) Tloc(b, BUNfirst(b)))[j] = *(int*)
PyArray_GETPTR1(pCol, j);
}
+ BATsetcount(b, cnt);
break;
}
// TODO: implement other types
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pyapi - Python API: More types
Changeset: 64236bddd072 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=64236bddd072 Added Files: monetdb5/extras/pyapi/Tests/pyapi00.stable.err monetdb5/extras/pyapi/Tests/pyapi00.stable.out monetdb5/extras/pyapi/Tests/pyapi02.malC Modified Files: monetdb5/extras/pyapi/Tests/All monetdb5/extras/pyapi/pyapi.c Branch: pyapi Log Message: Python API: More types diffs (truncated from 424 to 300 lines): diff --git a/monetdb5/extras/pyapi/Tests/All b/monetdb5/extras/pyapi/Tests/All --- a/monetdb5/extras/pyapi/Tests/All +++ b/monetdb5/extras/pyapi/Tests/All @@ -1,1 +1,2 @@ HAVE_LIBPY?pyapi00 +HAVE_LIBPY?pyapi02 diff --git a/monetdb5/extras/pyapi/Tests/pyapi00.stable.err b/monetdb5/extras/pyapi/Tests/pyapi00.stable.err new file mode 100644 --- /dev/null +++ b/monetdb5/extras/pyapi/Tests/pyapi00.stable.err @@ -0,0 +1,38 @@ +stderr of test 'pyapi00` in directory 'monetdb5/extras/pyapi` itself: + + +# 09:54:28 +# 09:54:28 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33990 --set mapi_usock=/var/tmp/mtest-8422/.s.monetdb.33990 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_monetdb5_extras_pyapi --set mal_listing=2 --set embedded_r=true --set embedded_py=true +# 09:54:28 + +# builtin opt gdk_dbpath = /Users/hannes/monetdb-install/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 33990 +# cmdline opt mapi_usock = /var/tmp/mtest-8422/.s.monetdb.33990 +# cmdline opt monet_prompt = +# cmdline opt mal_listing = 2 +# cmdline opt gdk_dbpath = /Users/hannes/monetdb-install/var/MonetDB/mTests_monetdb5_extras_pyapi +# cmdline opt mal_listing = 2 +# cmdline opt embedded_r = true +# cmdline opt embedded_py = true +# cmdline opt gdk_debug = 536870922 + +# 09:54:29 +# 09:54:29 mclient -lmal -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-8422 --port=33990 +# 09:54:29 + + +# 09:54:29 +# 09:54:29 Done. +# 09:54:29 + diff --git a/monetdb5/extras/pyapi/Tests/pyapi00.stable.out b/monetdb5/extras/pyapi/Tests/pyapi00.stable.out new file mode 100644 --- /dev/null +++ b/monetdb5/extras/pyapi/Tests/pyapi00.stable.out @@ -0,0 +1,63 @@ +stdout of test 'pyapi00` in directory 'monetdb5/extras/pyapi` itself: + + +# 09:54:28 +# 09:54:28 mserver5 --debug=10 --set gdk_nr_threads=0 --set mapi_open=true --set mapi_port=33990 --set mapi_usock=/var/tmp/mtest-8422/.s.monetdb.33990 --set monet_prompt= --forcemito --set mal_listing=2 --dbpath=/Users/hannes/monetdb-install/var/MonetDB/mTests_monetdb5_extras_pyapi --set mal_listing=2 --set embedded_r=true --set embedded_py=true +# 09:54:28 + +# MonetDB 5 server v11.20.0 +# This is an unreleased version +# Serving database 'mTests_monetdb5_extras_pyapi', using 4 threads +# Compiled for x86_64-apple-darwin14.1.0/64bit with 64bit OIDs and 128bit integers dynamically linked +# Found 16.000 GiB available main-memory. +# Copyright (c) 1993-July 2008 CWI. +# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved +# Visit http://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://dakar.da.cwi.nl:33990/ +# Listening for UNIX domain connection requests on mapi:monetdb:///var/tmp/mtest-8422/.s.monetdb.33990 +# MonetDB/GIS module loaded +# MonetDB/SQL module loaded +# MonetDB/Python module loaded +# MonetDB/R module loaded + +Ready. + +# 10:48:46 +# 10:48:46 mclient -lmal -ftest -Eutf-8 -i -e --host=/var/tmp/mtest-11109 --port=36417 +# 10:48:46 + +#io.print(b); +#--# +# ht # name +# void int # type +#--# +[ 0@0, 42 ] +[ 1@0, 43 ] +[ 2@0, 44 ] +[ 3@0, 45 ] +[ 4@0, 46 ] +[ 5@0, 47 ] +[ 6@0, 48 ] +[ 7@0, 49 ] +[ 8@0, 50 ] +[ 9@0, 51 ] +#io.print(r); +#--# +# ht # name +# void int # type +#--# +[ 0@0, 43 ] +[ 1@0, 44 ] +[ 2@0, 45 ] +[ 3@0, 46 ] +[ 4@0, 47 ] +[ 5@0, 48 ] +[ 6@0, 49 ] +[ 7@0, 50 ] +[ 8@0, 51 ] +[ 9@0, 52 ] + +# 09:54:29 +# 09:54:29 Done. +# 09:54:29 + diff --git a/monetdb5/extras/pyapi/Tests/pyapi02.malC b/monetdb5/extras/pyapi/Tests/pyapi02.malC new file mode 100644 --- /dev/null +++ b/monetdb5/extras/pyapi/Tests/pyapi02.malC @@ -0,0 +1,128 @@ +# input types testing + +# inty types + +bbit:= bat.new(:oid,:bit); +bat.append(bbit,1:bit); +bat.append(bbit,0:bit); +bat.append(bbit,1:bit); +bat.append(bbit,0:bit); +bat.append(bbit,nil:bit);
MonetDB: pyapi - Python API: mask array (working once, unclear why)
Changeset: aefe683f0b5e for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=aefe683f0b5e
Modified Files:
monetdb5/extras/pyapi/Tests/pyapi02.malC
monetdb5/extras/pyapi/pyapi.c
Branch: pyapi
Log Message:
Python API: mask array (working once, unclear why)
diffs (207 lines):
diff --git a/monetdb5/extras/pyapi/Tests/pyapi02.malC
b/monetdb5/extras/pyapi/Tests/pyapi02.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi02.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi02.malC
@@ -8,16 +8,16 @@ bat.append(bbit,0:bit);
bat.append(bbit,1:bit);
bat.append(bbit,0:bit);
bat.append(bbit,nil:bit);
-rbit:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),bbit);
+rbit:bat[:oid,:dbl] :=
pyapi.eval(nil:ptr,print(arg1)\nreturn([numpy.add(arg1.filled(0),42)]),bbit);
io.print(rbit);
bbte:= bat.new(:oid,:bte);
bat.append(bbte,42:bte);
bat.append(bbte,84:bte);
-bat.append(bbte,254:bte);
+bat.append(bbte,111:bte);
bat.append(bbte,0:bte);
bat.append(bbte,nil:bte);
-rbte:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),bbte);
+rbte:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([[42,43]]),bbte);
io.print(rbte);
bsht:= bat.new(:oid,:sht);
@@ -26,7 +26,7 @@ bat.append(bsht,82:sht);
bat.append(bsht,0:sht);
bat.append(bsht,3276:sht);
bat.append(bsht,nil:sht);
-rsht:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),bsht);
+rsht:bat[:oid,:dbl] :=
pyapi.eval(nil:ptr,return([numpy.add(arg1.filled(0),1)]),bsht);
io.print(rsht);
bint:= bat.new(:oid,:int);
@@ -35,7 +35,7 @@ bat.append(bint,846930886:int);
bat.append(bint,1681692777:int);
bat.append(bint,1714636915:int);
bat.append(bint,nil:int);
-rint:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),bint);
+rint:bat[:oid,:dbl] :=
pyapi.eval(nil:ptr,return([numpy.add(arg1.filled(0),1)]),bint);
io.print(rint);
bwrd:= bat.new(:oid,:wrd);
@@ -43,7 +43,7 @@ bat.append(bwrd,1804289383:wrd);
bat.append(bwrd,846930886:wrd);
bat.append(bwrd,1681692777:wrd);
bat.append(bwrd,1714636915:wrd);
-rwrd:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),bwrd);
+rwrd:bat[:oid,:dbl] :=
pyapi.eval(nil:ptr,return([numpy.add(arg1.filled(0),1)]),bwrd);
io.print(rwrd);
blng:= bat.new(:oid,:lng);
@@ -51,7 +51,7 @@ bat.append(blng,1804289383L);
bat.append(blng,846930886L);
bat.append(blng,1681692777L);
bat.append(blng,1714636915L);
-rlng:bat[:oid,:dbl] := pyapi.eval(nil:ptr,return([numpy.add(arg1,1)]),blng);
+rlng:bat[:oid,:dbl] :=
pyapi.eval(nil:ptr,return([numpy.add(arg1.filled(0),1)]),blng);
io.print(rlng);
# not sure what to with hge, numpy only supports 128 bits when sizeof(long)=16
@@ -89,7 +89,7 @@ bat.append(bstr,asdf:str);
bat.append(bstr,sd asd asd asd asd a:str);
bat.append(bstr,:str);
bat.append(bstr,nil:str);
-rstr:bat[:oid,:int] := rapi.eval(nil:ptr,unlist(lapply(arg1,nchar)),bstr);
+rstr:bat[:oid,:int] := pyapi.eval(nil:ptr,unlist(lapply(arg1,nchar)),bstr);
io.print(rstr);
@@ -103,26 +103,26 @@ bat.append(binto,1681692777:int);
bat.append(binto,1714636915:int);
bat.append(binto,nil:int);
-rintbi:bat[:oid,:int] := rapi.eval(nil:ptr,arg1,binto);
+rintbi:bat[:oid,:int] := pyapi.eval(nil:ptr,arg1,binto);
io.print(rintbi);
-rintbi2:bat[:oid,:int] := rapi.eval(nil:ptr,as.integer(arg1),binto);
+rintbi2:bat[:oid,:int] := pyapi.eval(nil:ptr,as.integer(arg1),binto);
io.print(rintbi2);
-rintbl:bat[:oid,:lng] := rapi.eval(nil:ptr,as.integer(arg1),binto);
+rintbl:bat[:oid,:lng] := pyapi.eval(nil:ptr,as.integer(arg1),binto);
io.print(rintbl);
-rintbh:bat[:oid,:hge] := rapi.eval(nil:ptr,as.integer(arg1),binto);
+rintbh:bat[:oid,:hge] := pyapi.eval(nil:ptr,as.integer(arg1),binto);
io.print(rintbh);
-rintbd:bat[:oid,:dbl] := rapi.eval(nil:ptr,as.numeric(arg1),binto);
+rintbd:bat[:oid,:dbl] := pyapi.eval(nil:ptr,as.numeric(arg1),binto);
io.print(rintbd);
-rintbs:bat[:oid,:str] := rapi.eval(nil:ptr,as.character(arg1),binto);
+rintbs:bat[:oid,:str] := pyapi.eval(nil:ptr,as.character(arg1),binto);
io.print(rintbs);
# factors should be strings
-rintbf:bat[:oid,:str] := rapi.eval(nil:ptr,as.factor(arg1),binto);
+rintbf:bat[:oid,:str] := pyapi.eval(nil:ptr,as.factor(arg1),binto);
io.print(rintbf);
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -58,6 +58,7 @@ static int pyapiInitialized = FALSE;
*(mtpe*) PyArray_GETPTR1(pCol, j); }
\
BATsetcount(bat, cnt); }
+// TODO: also handle the case if someone returns a masked array
#define _PYAPI_DEBUG_
@@ -152,6 +153,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
// for each input column (BAT):
for (i = pci-retc + 2; i pci-argc; i++) {
PyObject *vararray = NULL;
+ // null mask for masked array
// turn scalars into one-valued BATs
// TODO: also do
MonetDB: pyapi - Python API: user code running
Changeset: 03935b788de4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=03935b788de4
Modified Files:
monetdb5/extras/pyapi/pyapi.c
Branch: pyapi
Log Message:
Python API: user code running
diffs (53 lines):
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -243,7 +243,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
}
if (snprintf(pycall, pycalllen,
-def pyfun(%s):\n%s,
+import numpy\ndef pyfun(%s):\n%s,
argnames, expr_ind) = (int) pycalllen) {
msg = createException(MAL, pyapi.eval, Command too large);
goto wrapup;
@@ -256,16 +256,28 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
{
int pyret;
PyObject *pFunc, *pModule;
+
pModule = PyImport_Import(PyString_FromString(__main__));
pyret = PyRun_SimpleString(pycall);
pFunc = PyObject_GetAttrString(pModule, pyfun);
if (pyret != 0 || !pModule || !pFunc ||
!PyCallable_Check(pFunc)) {
msg = createException(MAL, pyapi.eval, could not
parse Python code %s, pycall);
+ goto wrapup;
+ }
+
+ pResult = PyObject_CallObject(pFunc, pArgs);
+ if (PyErr_Occurred()) {
+ PyObject *pErrType, *pErrVal, *pErrTb;
+ PyErr_Fetch(pErrType, pErrVal, pErrTb);
+ if (pErrVal) {
+ msg = createException(MAL, pyapi.eval,
Python exception: %s, PyString_AS_STRING(PyObject_Str(pErrVal)));
+ } else {
+ msg = createException(MAL, pyapi.eval,
Python exception: ?);
+ }
goto wrapup; // shudder
}
- pResult = PyObject_CallObject(pFunc, pArgs);
if (!pResult || !PyList_Check(pResult) || PyList_Size(pResult)
!= pci-retc) {
msg = createException(MAL, pyapi.eval, Invalid
result object. Need list of size %d containing numpy arrays, pci-retc);
goto wrapup;
@@ -335,10 +347,8 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
str PyAPIprelude(void *ret) {
(void) ret;
MT_lock_init(pyapiLock, pyapi_lock);
-
if (PyAPIEnabled()) {
MT_lock_set(pyapiLock, pyapi.evaluate);
- /* startup internal Python environment */
if (!pyapiInitialized) {
char* iar = NULL;
Py_Initialize();
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pyapi - Python API: user code indentation
Changeset: 80fa1acb9d42 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=80fa1acb9d42
Modified Files:
monetdb5/extras/pyapi/pyapi.c
Branch: pyapi
Log Message:
Python API: user code indentation
diffs (174 lines):
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -55,6 +55,9 @@ PyAPIevalAggr(Client cntxt, MalBlkPtr mb
return PyAPIeval(mb, stk, pci, 1);
}
+typedef enum {
+ NORMAL, SEENNL, INQUOTES, ESCAPED
+} pyapi_scan_state;
str PyAPIeval(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) {
sql_func * sqlfun = *(sql_func**) getArgReference(stk, pci, pci-retc);
@@ -64,8 +67,9 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
char argbuf[64];
char argnames[1000] = ;
size_t pos;
- char* rcall = NULL;
- size_t rcalllen;
+ char* pycall = NULL;
+ char *expr_ind = NULL;
+ size_t pycalllen, expr_ind_len;
str *args;
char *msg = MAL_SUCCEED;
BAT *b = NULL;
@@ -79,11 +83,14 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
pyapi_enableflag);
}
- rcalllen = strlen(exprStr) + sizeof(argnames) + 100;
- rcall = malloc(rcalllen);
+ pycalllen = strlen(exprStr) + sizeof(argnames) + 1000;
+ expr_ind_len = strlen(exprStr) + 1000;
+
+ pycall = GDKzalloc(pycalllen);
+ expr_ind =GDKzalloc(expr_ind_len);
args = (str*) GDKzalloc(sizeof(str) * pci-argc);
- if (args == NULL || rcall == NULL) {
+ if (args == NULL || pycall == NULL) {
throw(MAL, pyapi.eval, MAL_MALLOC_FAIL);
// TODO: free args and rcall
}
@@ -91,8 +98,6 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
// TODO: do we need this lock for Python as well?
MT_lock_set(pyapiLock, pyapi.evaluate);
-
-
// first argument after the return contains the pointer to the sql_func
structure
if (sqlfun != NULL sqlfun-ops-cnt 0) {
int carg = pci-retc + 2;
@@ -166,6 +171,7 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
PyTuple_SetItem(pArgs, ai++, vararray);
}
+ // create argument list
pos = 0;
for (i = pci-retc + 2; i pci-argc pos sizeof(argnames); i++) {
pos += snprintf(argnames + pos, sizeof(argnames) - pos, %s%s,
@@ -175,36 +181,90 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
msg = createException(MAL, pyapi.eval, Command too large);
goto wrapup;
}
- if (snprintf(rcall, rcalllen,
-ret - as.data.frame((function(%s){%s})(%s),
nm=NA, stringsAsFactors=F)\n,
-argnames, exprStr, argnames) = (int)
rcalllen) {
+
+ {
+ // indent every line in the expression by one level,
+ // if we find newline-tab, use tab, space otherwise
+ // two passes, first inserts null placeholder, second replaces
+ // need to be careful, newline might be in a quoted string
+ // this does not handle multi-line strings starting with
(yet?)
+ pyapi_scan_state state = SEENNL;
+ char indentchar = 0;
+ size_t py_pos, py_ind_pos = 0;
+ for (py_pos = 0; py_pos strlen(exprStr); py_pos++) {
+ // +1 because we need space for the \0 we append below.
+ if (py_ind_pos + 1 expr_ind_len) {
+ msg = createException(MAL, pyapi.eval,
Overflow in re-indentation);
+ goto wrapup;
+ }
+ switch(state) {
+ case NORMAL:
+ if (exprStr[py_pos] == '\'' ||
exprStr[py_pos] == '') {
+ state = INQUOTES;
+ }
+ if (exprStr[py_pos] == '\n') {
+ state = SEENNL;
+ }
+ break;
+
+ case INQUOTES:
+ if (exprStr[py_pos] == '\\') {
+ state = ESCAPED;
+ }
+ if (exprStr[py_pos] == '\'' ||
exprStr[py_pos] == '') {
+ state = NORMAL;
+ }
+ break;
+
+ case ESCAPED:
+ state = INQUOTES;
+ break;
+
+ case SEENNL:
+ if
MonetDB: pyapi - Python API mc
Changeset: 48953002afcc for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=48953002afcc
Modified Files:
monetdb5/extras/pyapi/Tests/pyapi00.malC
monetdb5/extras/pyapi/pyapi.c
monetdb5/mal/mal_listing.c
Branch: pyapi
Log Message:
Python API mc
diffs (69 lines):
diff --git a/monetdb5/extras/pyapi/Tests/pyapi00.malC
b/monetdb5/extras/pyapi/Tests/pyapi00.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi00.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi00.malC
@@ -12,7 +12,7 @@ bat.append(b,1189641421);
io.print(b);
-r:bat[:oid,:dbl] := pyapi.eval(nil:ptr,This string is somehow not arriving at
the C function, why?,b);
+r:bat[:oid,:int] := pyapi.eval(nil:ptr,int(arg1/1000),b);
io.print(r);
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -158,7 +158,8 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
//if ( v == int_nil)
//
PyList_SET_ITEM(varlist, j, );
//else
-
PyList_SET_ITEM(varlist, j, PyInt_FromLong(v));
+ // TODO: use numpy arrays here,
readonly, ignore NULLs for now?
+ PyList_SET_ITEM(varlist, j,
PyInt_FromLong(v));
}
break;
// TODO: implement other types
@@ -200,12 +201,12 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
PyObject *pFunc, *pModule, *pResult;
pModule = PyImport_Import(PyString_FromString(__main__));
- pyret = PyRun_SimpleString(def pyfun(x):\n return list(([e+1
for e in x],1)));
+ pyret = PyRun_SimpleString(def pyfun(x):\n print(x)\n return
list(([e+1 for e in x],1)));
pFunc = PyObject_GetAttrString(pModule, pyfun);
if (pyret != 0 || !pModule || !pFunc ||
!PyCallable_Check(pFunc)) {
// TODO: include parsed code
- msg = createException(MAL, pyapi.eval, could not
parse blubb);
+ msg = createException(MAL, pyapi.eval, could not
parse Python code %s, rcall);
goto wrapup; // shudder
}
@@ -216,16 +217,12 @@ str PyAPIeval(MalBlkPtr mb, MalStkPtr st
msg = createException(MAL, pyapi.eval, invalid
result object);
goto wrapup;
}
-
-
// delete the function again
PyRun_SimpleString(del pyfun);
-
}
// collect the return values
for (i = 0; i pci-retc; i++) {
- //SEXP ret_col = VECTOR_ELT(retval, i);
int bat_type = ATOMstorage(getColumnType(getArgType(mb,pci,i)));
cnt = (BUN) ret_rows;
diff --git a/monetdb5/mal/mal_listing.c b/monetdb5/mal/mal_listing.c
--- a/monetdb5/mal/mal_listing.c
+++ b/monetdb5/mal/mal_listing.c
@@ -96,7 +96,7 @@ renderTerm(MalBlkPtr mb, MalStkPtr stk,
len++;
}
showtype =closequote TYPE_str ||
((isVarUDFtype(mb,varid) || isVarTypedef(mb,varid)) isVarConstant(mb,varid))
||
- (isaBatType(getVarType(mb,varid) idx
p-retc));
+ (isaBatType(getVarType(mb,varid)) idx
p-retc);
if (stk isaBatType(getVarType(mb,varid))
abs(stk-stk[varid].val.ival) ){
BAT *d=
BBPquickdesc(abs(stk-stk[varid].val.ival),TRUE);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: pyapi - Python API: MAL level import
Changeset: fbc06a07b6e3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fbc06a07b6e3 Added Files: monetdb5/extras/pyapi/50_pyapi.mal monetdb5/extras/pyapi/Makefile.ag monetdb5/extras/pyapi/Tests/All monetdb5/extras/pyapi/Tests/pyapi00.malC monetdb5/extras/pyapi/pyapi.c monetdb5/extras/pyapi/pyapi.h monetdb5/extras/pyapi/pyapi.mal Modified Files: configure.ag monetdb5/extras/Makefile.ag testing/Mtest.py.in Branch: pyapi Log Message: Python API: MAL level import diffs (truncated from 600 to 300 lines): diff --git a/configure.ag b/configure.ag --- a/configure.ag +++ b/configure.ag @@ -226,6 +226,13 @@ AC_ARG_ENABLE(rintegration, enable_rintegration=$enableval, enable_rintegration=$dft_rintegration) +dft_pyintegration=auto +AC_ARG_ENABLE(pyintegration, + AS_HELP_STRING([--enable-pyintegration], + [enable support for Python integration into MonetDB (default=auto)]), + enable_pyintegration=$enableval, + enable_pyintegration=$dft_pyintegration) + dft_odbc=auto AC_ARG_ENABLE(odbc, AS_HELP_STRING([--enable-odbc], @@ -2211,6 +2218,7 @@ AC_SUBST(SPHINXCLIENT_CFLAGS, $SPHINXCLI AC_SUBST(SPHINXCLIENT_LIBS, $SPHINXCLIENT_LIBS) AM_CONDITIONAL(HAVE_SPHINXCLIENT, test x$have_sphinxclient != xno) +# R API (embedded R) have_libr=no if test x$enable_rintegration != xno; then case $enable_rintegration in @@ -2246,6 +2254,41 @@ if test x$enable_rintegration != xno; fi AM_CONDITIONAL(HAVE_LIBR, test x$have_libr != xno) +# Python API (embedded Python) +have_libpy=no +if test x$enable_pyintegration != xno; then + case $enable_pyintegration in + yes|auto) + XPATH=$PATH + ;; + /*) + XPATH=$enable_pyintegration + enable_pyintegration=yes + ;; + *) + AC_MSG_ERROR([--enable-pyintegration value must be yes|no|auto|absolute path of python-config]) + ;; + esac + AC_PATH_PROG(PYCMD,python-config,,$XPATH) + if test x$PYCMD = x; then + if test x$enable_pyintegration = xyes; then + AC_MSG_ERROR([python-config library required for Python integration support]) + else + have_libpy=no + why_have_libpy=(python-config command not found) + enable_pyintegration=no + disable_pyintegration=(python-config command not found) + fi + elif libpy_CFLAGS=`$PYCMD --cflags ` libpy_LIBS=`$PYCMD --ldflags`; then + have_libpy=yes + AC_DEFINE(HAVE_LIBPY, 1, [Define if we can link to python]) + AC_SUBST(libpy_CFLAGS, $libpy_CFLAGS) + AC_SUBST(libpy_LIBS, $libpy_LIBS) + fi +fi +AM_CONDITIONAL(HAVE_LIBPY, test x$have_libpy != xno) + + # ODBC, only used by ODBC driver if test x$enable_odbc != xno; then have_unixodbc=auto @@ -3270,25 +3313,26 @@ done echo echo * Enabled/disabled components: for comp in \ - 'gdk ' \ - 'monetdb5' \ - 'sql ' \ - 'geom' \ - 'console ' \ - 'gsl ' \ - 'fits' \ - 'jsonstore ' \ - 'microhttpd ' \ - 'rintegration' \ - 'odbc' \ - 'jdbc' \ - 'control ' \ - 'testing ' \ - 'developer ' \ - 'strict ' \ - 'debug ' \ - 'assert ' \ - 'optimize' \ + 'gdk ' \ + 'monetdb5 ' \ + 'sql ' \ + 'geom ' \ + 'console ' \ + 'gsl ' \ + 'fits ' \ + 'jsonstore' \ + 'microhttpd ' \ + 'rintegration ' \ + 'pyintegration' \ + 'odbc ' \ + 'jdbc ' \ + 'control ' \ + 'testing ' \ + 'developer' \ + 'strict ' \ + 'debug' \ + 'assert ' \ + 'optimize ' \ ; do eval dft=\$dft_$comp eval enable=\$enable_$comp diff --git a/monetdb5/extras/Makefile.ag b/monetdb5/extras/Makefile.ag --- a/monetdb5/extras/Makefile.ag +++ b/monetdb5/extras/Makefile.ag @@ -4,5 +4,5 @@ # # Copyright 2008-2015 MonetDB B.V. -SUBDIRS = HAVE_SPHINXCLIENT?sphinx mal_optimizer_template HAVE_LIBR?rapi +SUBDIRS = HAVE_SPHINXCLIENT?sphinx mal_optimizer_template HAVE_LIBR?rapi HAVE_LIBPY?pyapi diff --git a/monetdb5/extras/pyapi/50_pyapi.mal b/monetdb5/extras/pyapi/50_pyapi.mal new file mode 100644 --- /dev/null +++ b/monetdb5/extras/pyapi/50_pyapi.mal @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright 2008-2015 MonetDB
MonetDB: pyapi - Python API:
Changeset: 663b509eff6c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=663b509eff6c
Modified Files:
monetdb5/extras/pyapi/Makefile.ag
monetdb5/extras/pyapi/Tests/pyapi00.malC
monetdb5/extras/pyapi/pyapi.c
monetdb5/extras/pyapi/pyapi.mal
Branch: pyapi
Log Message:
Python API:
diffs (truncated from 508 to 300 lines):
diff --git a/monetdb5/extras/pyapi/Makefile.ag
b/monetdb5/extras/pyapi/Makefile.ag
--- a/monetdb5/extras/pyapi/Makefile.ag
+++ b/monetdb5/extras/pyapi/Makefile.ag
@@ -18,19 +18,18 @@ lib__pyapi = {
MODULE
DIR = libdir/monetdb5
SOURCES = pyapi.c pyapi.h
+ XDEPS = $(libpy_LIBDEP)
LIBS = ../../tools/libmonetdb5 \
../../../gdk/libbat \
$(MALLOC_LIBS) $(libpy_LIBS)
}
-
headers_pyapi_mal = {
HEADERS = mal
DIR = libdir/monetdb5
SOURCES = pyapi.mal
}
-
headers_autoload = {
HEADERS = mal
DIR = libdir/monetdb5/autoload
diff --git a/monetdb5/extras/pyapi/Tests/pyapi00.malC
b/monetdb5/extras/pyapi/Tests/pyapi00.malC
--- a/monetdb5/extras/pyapi/Tests/pyapi00.malC
+++ b/monetdb5/extras/pyapi/Tests/pyapi00.malC
@@ -12,7 +12,7 @@ bat.append(b,1189641421);
io.print(b);
-r:bat[:oid,:dbl] := pyapi.eval(nil:ptr,TODO,b);
+r:bat[:oid,:dbl] := pyapi.eval(nil:ptr,print(arg1);someval - Re(fft(arg1));
print(someval); return(someval);,b);
io.print(r);
diff --git a/monetdb5/extras/pyapi/pyapi.c b/monetdb5/extras/pyapi/pyapi.c
--- a/monetdb5/extras/pyapi/pyapi.c
+++ b/monetdb5/extras/pyapi/pyapi.c
@@ -28,16 +28,14 @@ int PyAPIEnabled(void) {
|| GDKgetenv_isyes(pyapi_enableflag));
}
+// TODO: exclude pyapi from mergetable, too
// TODO: can we call the Python interpreter in a multi-thread environment?
static MT_Lock pyapiLock;
static int pyapiInitialized = FALSE;
-static int PyAPIinitialize(void) {
- Py_Initialize();
- pyapiInitialized++;
- return 0;
-}
+#define _PYAPI_DEBUG_
+
pyapi_export str PyAPIevalStd(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci) {
@@ -48,212 +46,214 @@ pyapi_export str PyAPIevalAggr(Client cn
return PyAPIeval(cntxt, mb, stk, pci, 1);
}
+
str PyAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit
grouped) {
+ sql_func * sqlfun = *(sql_func**) getArgReference(stk, pci, pci-retc);
+ str exprStr = *getArgReference_str(stk, pci, pci-retc + 1);
- sql_func * sqlfun = *(sql_func**) getArgReference(stk, pci, pci-retc);
- str exprStr = *getArgReference_str(stk, pci, pci-retc + 1);
+ int i = 1, ai = 0;
+ char argbuf[64];
+ char argnames[1000] = ;
+ size_t pos;
+ char* rcall = NULL;
+ size_t rcalllen;
+ size_t ret_rows = 0;
+ //int ret_cols = 0; /* int because pci-retc is int, too*/
+ str *args;
+ //int evalErr;
+ char *msg = MAL_SUCCEED;
+ BAT *b;
+ BUN cnt;
+ node * argnode;
+ int seengrp = FALSE;
+ PyObject *pArgs; // this is going to be the parameter tuple
- int i = 1, ai = 0;
- char argbuf[64];
- char argnames[1000] = ;
- size_t pos;
- char* rcall = NULL;
- size_t rcalllen;
- size_t ret_rows = 0;
- //int ret_cols = 0; /* int because pci-retc is int, too*/
- str *args;
- //int evalErr;
- char *msg = MAL_SUCCEED;
- BAT *b;
- BUN cnt;
- node * argnode;
- int seengrp = FALSE;
- PyObject *pArgs; // this is going to be the parameter tuple!
+ // we don't need no context, but the compiler needs us to touch it (...)
+ (void) cntxt;
- // we don't need no context, but the compiler needs us to touch
it (...)
- (void) cntxt;
+ if (!PyAPIEnabled()) {
+ throw(MAL, pyapi.eval,
+ Embedded Python has not been enabled. Start server
with --set %s=true,
+ pyapi_enableflag);
+ }
- if (!PyAPIEnabled()) {
- throw(MAL, pyapi.eval,
- Embedded Python has not been enabled. Start
server with --set %s=true,
- pyapi_enableflag);
+ rcalllen = strlen(exprStr) + sizeof(argnames) + 100;
+ rcall = malloc(rcalllen);
+ args = (str*) GDKzalloc(sizeof(str) * pci-argc);
+
+ if (args == NULL || rcall == NULL) {
+ throw(MAL, pyapi.eval, MAL_MALLOC_FAIL);
+ // TODO: free args and rcall
+ }
+
+ // TODO: do we need this lock for Python as well?
+ MT_lock_set(pyapiLock, pyapi.evaluate);
+
+
+
+ // first argument after the return contains the pointer to the sql_func
structure
+ if (sqlfun
