Changeset: 71c0440590e1 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=71c0440590e1 Added Files: tools/embeddedpy/pyclient.c tools/embeddedpy/pyclient.h Modified Files: monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h tools/embeddedpy/embedded_module.c tools/embeddedpy/embeddedpy.c tools/embeddedpy/embeddedpy.h Branch: pyapi Log Message:
Added monetdblite.connect() support, allowing users to open multiple client connections with monetdblite. Also free the GIL when a monetdblite query is running. diffs (truncated from 798 to 300 lines): diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c --- a/monetdb5/mal/mal_client.c +++ b/monetdb5/mal/mal_client.c @@ -69,6 +69,7 @@ MCinit(void) { char *max_clients = GDKgetenv("max_clients"); int maxclients = 0; + Client c; if (max_clients != NULL) maxclients = atoi(max_clients); @@ -85,6 +86,9 @@ MCinit(void) showException(GDKout, MAL, "MCinit",MAL_MALLOC_FAIL); mal_exit(); } + for (c = mal_clients; c < mal_clients + MAL_MAXCLIENTS; c++) { + MT_lock_init(&c->query_lock, "client.query_lock"); + } } int diff --git a/monetdb5/mal/mal_client.h b/monetdb5/mal/mal_client.h --- a/monetdb5/mal/mal_client.h +++ b/monetdb5/mal/mal_client.h @@ -165,6 +165,15 @@ typedef struct CLIENT { void *sqlcontext; /* + Executing multiple SQL queries in parallel with the same client + does not work because of shared client state, causing + segfaults. In normal usage this is not possible, but + in MonetDBLite and MonetDB/Python this can happen. To + prevent segfualts, we have to lock the client when performing a query. + */ + MT_Lock query_lock; + + /* * keep track of which instructions are currently being executed */ bit active; /* processing a query or not */ diff --git a/tools/embeddedpy/embedded_module.c b/tools/embeddedpy/embedded_module.c --- a/tools/embeddedpy/embedded_module.c +++ b/tools/embeddedpy/embedded_module.c @@ -22,12 +22,15 @@ static char create_docstring[] = "monetdblite.create(tablename, dictionary), monetdblite.create(tablename, column_names, values) => Create a SQL table from the given Python objects, objects must either be a (column name, value) dictionary or a list of column names and a list of values"; static char insert_docstring[] = "monetdblite.insert(tablename, dictionary), monetdblite.insert(tablename, column_names, values) => Insert a set of values into a SQL table"; +static char connect_docstring[] = + "monetdblite.connect() => Create a connection object to the monetdblite database."; static PyMethodDef module_methods[] = { {"init", monetdb_init, METH_O, init_docstring}, - {"sql", monetdb_sql, METH_O, sql_docstring}, - {"create", monetdb_create, METH_VARARGS, create_docstring}, - {"insert", monetdb_insert, METH_VARARGS, insert_docstring}, + {"sql", (PyCFunction)monetdb_sql, METH_VARARGS|METH_KEYWORDS, sql_docstring}, + {"create", (PyCFunction)monetdb_create, METH_VARARGS|METH_KEYWORDS, create_docstring}, + {"insert", (PyCFunction)monetdb_insert, METH_VARARGS|METH_KEYWORDS, insert_docstring}, + {"connect", (PyCFunction)monetdb_client, METH_NOARGS, connect_docstring}, {NULL, NULL, 0, NULL} }; @@ -40,5 +43,5 @@ PyMODINIT_FUNC initmonetdblite(void) return; //import numpy stuff - init_embedded_py(); + monetdblite_init(); } diff --git a/tools/embeddedpy/embeddedpy.c b/tools/embeddedpy/embeddedpy.c --- a/tools/embeddedpy/embeddedpy.c +++ b/tools/embeddedpy/embeddedpy.c @@ -32,23 +32,8 @@ #include "pyapi.h" #include "pytypes.h" -///////////////////////////////////////////////////////////////////////////////////////////////// -// copy paste from embedded.h -typedef struct append_data { - char* colname; - bat batid; -} append_data; - -str monetdb_startup(char* dir, char silent); -str monetdb_query(char* query, void** result); -str monetdb_create_table(char *schema, char *table_name, append_data *ad, int ncols); -str monetdb_append(const char* schema, const char* table, append_data *ad, int ncols); -void monetdb_cleanup_result(void* output); -static str monetdb_get_columns(const char* schema_name, const char *table_name, int *column_count, char ***column_names, int **column_types, sql_subtype ***sql_subtypes); - -static bit monetdb_embedded_initialized = 0; -static MT_Lock monetdb_embedded_lock; -///////////////////////////////////////////////////////////////////////////////////////////////// +#include "pyclient.h" +#include "embedded.h" PyObject *monetdb_init(PyObject *self, PyObject *args) { @@ -61,12 +46,14 @@ PyObject *monetdb_init(PyObject *self, P { char *msg; char *directory = &(((PyStringObject*)args)->ob_sval[0]); + char installdir[1024]; printf("Making directory %s\n", directory); if (GDKcreatedir(directory) != GDK_SUCCEED) { PyErr_Format(PyExc_Exception, "Failed to create directory %s.", directory); return NULL; } - msg = monetdb_startup(directory, 1); + snprintf(installdir, 1024, "%s/../", BINDIR); + msg = monetdb_startup(installdir, directory, 1); if (msg != MAL_SUCCEED) { PyErr_Format(PyExc_Exception, "Failed to initialize MonetDB. %s", msg); return NULL; @@ -76,26 +63,40 @@ PyObject *monetdb_init(PyObject *self, P Py_RETURN_NONE; } -PyObject *monetdb_sql(PyObject *self, PyObject *args) +PyObject *monetdb_sql(PyObject *self, PyObject *args, PyObject *keywds) { + Client c = monetdb_default_client; + char *query; + PyObject *client = NULL; + static char *kwlist[] = {"query", "conn", NULL}; (void) self; - if (!PyString_CheckExact(args)) { - PyErr_SetString(PyExc_TypeError, "Expected a SQL query as a single string argument."); - return NULL; - } if (!monetdb_embedded_initialized) { PyErr_SetString(PyExc_Exception, "monetdb has not been initialized yet"); return NULL; } - + if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|O", kwlist, &query, &client)) { + return NULL; + } + if (client != NULL) { + if (!PyClient_CheckExact(client)) { + PyErr_SetString(PyExc_Exception, "conn must be a connection object created by monetdblite.connect()."); + return NULL; + } + c = ((PyClientObject*)client)->cntxt; + } { PyObject *result; res_table* output = NULL; + PyObject *querystring; char* err; // Append ';'' to the SQL query, just in case - args = PyString_FromFormat("%s;", &(((PyStringObject*)args)->ob_sval[0])); + querystring = PyString_FromFormat("%s;", query); // Perform the SQL query - err = monetdb_query(&(((PyStringObject*)args)->ob_sval[0]), (void**)&output); +Py_BEGIN_ALLOW_THREADS + MT_lock_set(&c->query_lock, "client.query_lock"); + err = monetdb_query(c, &(((PyStringObject*)querystring)->ob_sval[0]), (void**)&output); + MT_lock_unset(&c->query_lock, "client.query_lock"); +Py_END_ALLOW_THREADS if (err != NULL) { PyErr_Format(PyExc_Exception, "SQL Query Failed: %s", (err ? err : "<no error>")); return NULL; @@ -117,15 +118,15 @@ PyObject *monetdb_sql(PyObject *self, Py input.scalar = false; input.sql_subtype = &col.type; - numpy_array = PyMaskedArray_FromBAT(&mal_clients[0], &input, 0, input.count, &msg, true); + numpy_array = PyMaskedArray_FromBAT(c, &input, 0, input.count, &msg, true); if (!numpy_array) { - monetdb_cleanup_result(output); + monetdb_cleanup_result(c, output); PyErr_Format(PyExc_Exception, "SQL Query Failed: %s", (msg ? msg : "<no error>")); return NULL; } PyDict_SetItem(result, PyString_FromString(output->cols[i].name), numpy_array); } - monetdb_cleanup_result(output); + monetdb_cleanup_result(c, output); return result; } else { Py_RETURN_NONE; @@ -133,40 +134,49 @@ PyObject *monetdb_sql(PyObject *self, Py } } -PyObject *monetdb_create(PyObject *self, PyObject *args) +PyObject *monetdb_create(PyObject *self, PyObject *args, PyObject *keywds) { - char *schema = "sys"; + char *schema_name = "sys"; char *table_name; - PyObject *dict = NULL, *values = NULL; + PyObject *values = NULL, *client = NULL, *colnames = NULL; + Client c = monetdb_default_client; + static char *kwlist[] = {"name", "values", "colnames", "schema", "conn", NULL}; int i; (void) self; if (!monetdb_embedded_initialized) { PyErr_SetString(PyExc_Exception, "monetdb has not been initialized yet"); return NULL; } - if (!PyArg_ParseTuple(args, "sO|O", &table_name, &dict, &values)) { + if (!PyArg_ParseTupleAndKeywords(args, keywds, "sO|OsO", kwlist, &table_name, &values, &colnames, &schema_name, &client)) { return NULL; } - if (values == NULL) { - if (!PyDict_CheckExact(dict)) { - PyErr_SetString(PyExc_TypeError, "the second argument has to be a dict"); + if (client != NULL) { + if (!PyClient_CheckExact(client)) { + PyErr_SetString(PyExc_Exception, "conn must be a connection object created by monetdblite.connect()."); + return NULL; + } + c = ((PyClientObject*)client)->cntxt; + } + if (colnames == NULL) { + if (!PyDict_CheckExact(values)) { + PyErr_SetString(PyExc_TypeError, "no colnames are specified and values is not a dict"); return NULL; } PyErr_Format(PyExc_Exception, "dict is not implemented yet"); return NULL; Py_RETURN_NONE; } else { - if (!PyList_CheckExact(dict)) { - PyErr_SetString(PyExc_TypeError, "the second argument (column names) must be a list"); + if (!PyList_Check(colnames)) { + PyErr_SetString(PyExc_TypeError, "colnames must be a list"); return NULL; } - if (PyList_Size(dict) == 0) { - PyErr_SetString(PyExc_TypeError, "the list of column names must be non-zero"); + if (PyList_Size(colnames) == 0) { + PyErr_SetString(PyExc_TypeError, "colnames must have at least one element"); return NULL; } - for(i = 0; i < PyList_Size(dict); i++) { - PyObject *column_name = PyList_GetItem(dict, i); + for(i = 0; i < PyList_Size(colnames); i++) { + PyObject *column_name = PyList_GetItem(colnames, i); if (!PyString_CheckExact(column_name)) { PyErr_Format(PyExc_TypeError, "the entry %d in the column names is not a string", i); return NULL; @@ -176,7 +186,7 @@ PyObject *monetdb_create(PyObject *self, { char *msg = NULL; PyObject *pResult; - int columns = PyList_Size(dict); + int columns = PyList_Size(colnames); append_data *append_bats = NULL; PyReturn *pyreturn_values = NULL; @@ -195,9 +205,13 @@ PyObject *monetdb_create(PyObject *self, BAT *b = PyObject_ConvertToBAT(&pyreturn_values[i], NULL, PyType_ToBat(pyreturn_values[i].result_type), i, 0, &msg, true); if (b == NULL) goto cleanup; append_bats[i].batid = b->batCacheid; - append_bats[i].colname = PyString_AS_STRING(PyList_GetItem(dict, i)); + append_bats[i].colname = PyString_AS_STRING(PyList_GetItem(colnames, i)); } - msg = monetdb_create_table(schema, table_name, append_bats, columns); +Py_BEGIN_ALLOW_THREADS + MT_lock_set(&c->query_lock, "client.query_lock"); + msg = monetdb_create_table(c, schema_name, table_name, append_bats, columns); + MT_lock_unset(&c->query_lock, "client.query_lock"); +Py_END_ALLOW_THREADS cleanup: if (pyreturn_values) GDKfree(pyreturn_values); if (append_bats) { @@ -216,19 +230,29 @@ cleanup: Py_RETURN_NONE; } -PyObject *monetdb_insert(PyObject *self, PyObject *args) +PyObject *monetdb_insert(PyObject *self, PyObject *args, PyObject *keywds) { char *schema_name = "sys"; char *table_name; - PyObject *values = NULL; + PyObject *values = NULL, *client = NULL; + Client c = monetdb_default_client; + static char *kwlist[] = {"name", "values", "schema", "conn", NULL}; (void) self; if (!monetdb_embedded_initialized) { PyErr_SetString(PyExc_Exception, "monetdb has not been initialized yet"); return NULL; } - if (!PyArg_ParseTuple(args, "sO", &table_name, &values)) { + + if (!PyArg_ParseTupleAndKeywords(args, keywds, "sO|sO", kwlist, &table_name, &values, &schema_name, &client)) { return NULL; } + if (client != NULL) { + if (!PyClient_CheckExact(client)) { + PyErr_SetString(PyExc_Exception, "conn must be a connection object created by monetdblite.connect()."); + return NULL; + } + c = ((PyClientObject*)client)->cntxt; + } _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list