Changeset: c46105552cd2 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c46105552cd2
Modified Files:
monetdb5/extras/jaql/Tests/All
monetdb5/extras/jaql/Tests/json02.mal
monetdb5/extras/jaql/Tests/json02.stable.err
monetdb5/extras/jaql/Tests/json02.stable.out
monetdb5/extras/jaql/json.c
monetdb5/extras/jaql/json.h
monetdb5/extras/jaql/json.mal
Branch: jacqueline
Log Message:
json: add extract functionality
extract produces a new JSON object with the given list of element ids
from the kind BAT of the given JSON object.
This is useful for cleaning up a JSON object after many modifications
(calculations, transforms, filters) using elem 0@0, or for
pre-extracting some elements with a different starting oid. The join
operation for instance can use this to copy in the necessary bits into
another JSON object.
diffs (298 lines):
diff --git a/monetdb5/extras/jaql/Tests/All b/monetdb5/extras/jaql/Tests/All
--- a/monetdb5/extras/jaql/Tests/All
+++ b/monetdb5/extras/jaql/Tests/All
@@ -1,5 +1,6 @@
json00
json01
+json02
expand00
filter00
diff --git a/monetdb5/extras/jaql/Tests/json02.mal
b/monetdb5/extras/jaql/Tests/json02.mal
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/json02.mal
@@ -0,0 +1,31 @@
+# test extraction (cleanup)
+
+s := io.stdout();
+
+(j1,j2,j3,j4,j5,j6,j7) :=
json.shred("[\"hello\",{\"world\":[0,2,true,null]}]");
+json.print(s, j1,j2,j3,j4,j5,j6,j7);
+
+elem0 := bat.new(:oid, :oid);
+elem0 := bat.insert(elem0, 0@0, 0@0);
+
+(k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem0,0@0);
+json.print(s, k1,k2,k3,k4,k5,k6,k7);
+
+elem1 := bat.new(:oid, :oid);
+elem1 := bat.insert(elem1, 0@0, 1@0);
+
+(k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem1,0@0);
+json.print(s, k1,k2,k3,k4,k5,k6,k7);
+
+elem2 := bat.new(:oid, :oid);
+elem2 := bat.insert(elem2, 0@0, 2@0);
+
+(k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem2,0@0);
+json.print(s, k1,k2,k3,k4,k5,k6,k7);
+
+elem3 := bat.new(:oid, :oid);
+elem3 := bat.insert(elem3, 0@0, 1@0);
+elem3 := bat.insert(elem3, 0@0, 3@0);
+
+(k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem3,0@0);
+json.print(s, k1,k2,k3,k4,k5,k6,k7);
diff --git a/monetdb5/extras/jaql/Tests/json02.stable.err
b/monetdb5/extras/jaql/Tests/json02.stable.err
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/json02.stable.err
@@ -0,0 +1,31 @@
+stderr of test 'json02` in directory 'extras/jaql` itself:
+
+
+# 22:47:09 >
+# 22:47:09 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm"
"--set" "mapi_open=true" "--set" "mapi_port=33699" "--set" "monet_prompt="
"--trace" "--forcemito" "--set" "mal_listing=2" "--dbname=mTests_extras_jaql"
"json02.mal"
+# 22:47:09 >
+
+# builtin opt gdk_dbname = demo
+# builtin opt gdk_dbfarm =
/ufs/fabian/scratch/ssd/monetdb/jacqueline/program-x86_64/var/lib/monetdb5/dbfarm
+# builtin opt gdk_debug = 0
+# builtin opt gdk_alloc_map = no
+# builtin opt gdk_vmtrim = yes
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt gdk_dbfarm =
/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 33699
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbname = mTests_extras_jaql
+
+# 22:47:10 >
+# 22:47:10 > "Done."
+# 22:47:10 >
+
diff --git a/monetdb5/extras/jaql/Tests/json02.stable.out
b/monetdb5/extras/jaql/Tests/json02.stable.out
new file mode 100644
--- /dev/null
+++ b/monetdb5/extras/jaql/Tests/json02.stable.out
@@ -0,0 +1,51 @@
+stdout of test 'json02` in directory 'extras/jaql` itself:
+
+
+# 22:47:09 >
+# 22:47:09 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"gdk_dbfarm=/net/sofia.ins.cwi.nl/export/scratch1/fabian/tmp/mtest-jacqueline-sofia.ins.cwi.nl/five/dbfarm"
"--set" "mapi_open=true" "--set" "mapi_port=33699" "--set" "monet_prompt="
"--trace" "--forcemito" "--set" "mal_listing=2" "--dbname=mTests_extras_jaql"
"json02.mal"
+# 22:47:09 >
+
+# MonetDB 5 server v11.8.0 "jacqueline-0c67cb41e11b+"
+# Serving database 'mTests_extras_jaql', using 8 threads
+# Compiled for x86_64-pc-linux-gnu/64bit with 64bit OIDs dynamically linked
+# Found 15.662 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2012 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://sofia.ins.cwi.nl:33699/
+# MonetDB/GIS module loaded
+# MonetDB/SQL module loaded
+# MonetDB/DataCell module not loaded: MALException:jaql.context:JAQL
environment not found
+function user.main():void;
+# test extraction (cleanup)
+ s := io.stdout();
+ (j1,j2,j3,j4,j5,j6,j7) :=
json.shred("[\"hello\",{\"world\":[0,2,true,null]}]");
+ json.print(s,j1,j2,j3,j4,j5,j6,j7);
+ elem0 := bat.new(:oid,:oid);
+ elem0 := bat.insert(elem0,0@0,0@0);
+ (k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem0,0@0);
+ json.print(s,k1,k2,k3,k4,k5,k6,k7);
+ elem1 := bat.new(:oid,:oid);
+ elem1 := bat.insert(elem1,0@0,1@0);
+ (k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem1,0@0);
+ json.print(s,k1,k2,k3,k4,k5,k6,k7);
+ elem2 := bat.new(:oid,:oid);
+ elem2 := bat.insert(elem2,0@0,2@0);
+ (k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem2,0@0);
+ json.print(s,k1,k2,k3,k4,k5,k6,k7);
+ elem3 := bat.new(:oid,:oid);
+ elem3 := bat.insert(elem3,0@0,1@0);
+ elem3 := bat.insert(elem3,0@0,3@0);
+ (k1,k2,k3,k4,k5,k6,k7) := json.extract(j1,j2,j3,j4,j5,j6,j7,elem3,0@0);
+ json.print(s,k1,k2,k3,k4,k5,k6,k7);
+end main;
+[ "hello", { "world": [ 0, 2, true, null ] } ]
+[ [ "hello", { "world": [ 0, 2, true, null ] } ] ]
+[ "hello" ]
+[ { "world": [ 0, 2, true, null ] } ]
+[ "hello", [ 0, 2, true, null ] ]
+
+# 22:47:10 >
+# 22:47:10 > "Done."
+# 22:47:10 >
+
diff --git a/monetdb5/extras/jaql/json.c b/monetdb5/extras/jaql/json.c
--- a/monetdb5/extras/jaql/json.c
+++ b/monetdb5/extras/jaql/json.c
@@ -724,3 +724,134 @@ JSONdrop(int *ret, str *name)
*ret = 0;
return MAL_SUCCEED;
}
+
+oid
+json_copy_entry(BATiter bik, BATiter bis, BATiter bii, BATiter bid, BATiter
bia, BATiter bio, BATiter bin, oid start, oid v, jsonbat *jb, jsonbat *jbr)
+{
+ oid w, x;
+ chr k;
+
+ BUNfndOID(w, bik, &v);
+ k = *(chr *)BUNtail(bik, w);
+ BUNappend(jbr->kind, &k, FALSE);
+
+ w = BUNlast(jbr->kind) - 1 + start;
+ switch (k) {
+ case 'i':
+ BUNfndOID(x, bii, &v);
+ BUNins(jbr->integer, &w, BUNtail(bii, x), FALSE);
+ break;
+ case 'd':
+ BUNfndOID(x, bid, &v);
+ BUNins(jbr->doble, &w, BUNtail(bid, x), FALSE);
+ break;
+ case 's':
+ BUNfndOID(x, bis, &v);
+ BUNins(jbr->string, &w, BUNtail(bis, x), FALSE);
+ break;
+ case 'n':
+ case 't':
+ case 'f':
+ /* nothing to do here */
+ break;
+ case 'o': {
+ BAT *elems;
+ BUN p, q;
+ BATiter bi;
+ oid y, z;
+ elems = BATmirror(BATselect(BATmirror(jb->object), &v,
&v));
+ bi = bat_iterator(elems);
+ BATloop (elems, p, q) {
+ x = *(oid *)BUNtail(bi, p);
+ z = json_copy_entry(bik, bis, bii, bid, bia,
bio, bin,
+ start, x, jb, jbr);
+ BUNins(jbr->object, &w, &z, FALSE);
+ BUNfndOID(y, bin, &x);
+ BUNins(jbr->name, &z, BUNtail(bin, y), FALSE);
+ }
+ BBPunfix(elems->batCacheid);
+ break;
+ }
+ case 'a': {
+ BAT *elems;
+ BUN p, q;
+ BATiter bi;
+ oid z;
+ elems = BATmirror(BATselect(BATmirror(jb->array), &v,
&v));
+ bi = bat_iterator(elems);
+ BATloop (elems, p, q) {
+ z = json_copy_entry(bik, bis, bii, bid, bia,
bio, bin,
+ start, *(oid *)BUNtail(bi, p),
jb, jbr);
+ BUNins(jbr->array, &w, &z, FALSE);
+ }
+ BBPunfix(elems->batCacheid);
+ break;
+ }
+ }
+
+ return w;
+}
+
+str
+JSONextract(int *rkind, int *rstring, int *rinteger, int *rdoble, int *rarray,
int *robject, int *rname, int *kind, int *string, int *integer, int *doble, int
*array, int *object, int *name, int *elems, oid *startoid)
+{
+ jsonbat jb, jbr;
+ BAT *e;
+ BATiter bie, bik, bis, bii, bid, bia, bio, bin;
+ BUN p, q;
+ oid v, w, z;
+
+ loadbats();
+ bik = bat_iterator(jb.kind);
+ bis = bat_iterator(jb.string);
+ bii = bat_iterator(jb.integer);
+ bid = bat_iterator(jb.doble);
+ bia = bat_iterator(jb.array);
+ bio = bat_iterator(jb.object);
+ bin = bat_iterator(jb.name);
+ e = BBPquickdesc(ABS(*elems), FALSE);
+ if (*elems < 0)
+ e = BATmirror(e);
+ BBPfix(*elems);
+ bie = bat_iterator(e);
+
+ memset(&jbr, 0, sizeof(jsonbat));
+
+ /* initialise all bats */
+ jbr.kind = BATnew(TYPE_void, TYPE_chr, BATTINY);
+ jbr.kind = BATseqbase(jbr.kind, *startoid);
+ jbr.string = BATnew(TYPE_oid, TYPE_str, BATTINY);
+ jbr.doble = BATnew(TYPE_oid, TYPE_dbl, BATTINY);
+ jbr.integer = BATnew(TYPE_oid, TYPE_lng, BATTINY);
+ jbr.name = BATnew(TYPE_oid, TYPE_str, BATTINY);
+ jbr.object = BATnew(TYPE_oid, TYPE_oid, BATTINY);
+ jbr.array = BATnew(TYPE_oid, TYPE_oid, BATTINY);
+
+ /* return all elems as the outermost array */
+ BUNappend(jbr.kind, "a", FALSE);
+ w = BUNlast(jbr.kind) - 1 + *startoid;
+ BATloop(e, p, q) {
+ v = *(oid *)BUNtail(bie, p);
+ z = json_copy_entry(bik, bis, bii, bid, bia, bio, bin,
+ *startoid, v, &jb, &jbr);
+ BUNins(jbr.array, &w, &z, FALSE);
+ }
+
+ unloadbats();
+
+ BBPkeepref(jbr.kind->batCacheid);
+ *rkind = jbr.kind->batCacheid;
+ BBPkeepref(jbr.string->batCacheid);
+ *rstring = jbr.string->batCacheid;
+ BBPkeepref(jbr.integer->batCacheid);
+ *rinteger = jbr.integer->batCacheid;
+ BBPkeepref(jbr.doble->batCacheid);
+ *rdoble = jbr.doble->batCacheid;
+ BBPkeepref(jbr.array->batCacheid);
+ *rarray = jbr.array->batCacheid;
+ BBPkeepref(jbr.object->batCacheid);
+ *robject = jbr.object->batCacheid;
+ BBPkeepref(jbr.name->batCacheid);
+ *rname = jbr.name->batCacheid;
+ return MAL_SUCCEED;
+}
diff --git a/monetdb5/extras/jaql/json.h b/monetdb5/extras/jaql/json.h
--- a/monetdb5/extras/jaql/json.h
+++ b/monetdb5/extras/jaql/json.h
@@ -40,6 +40,7 @@ json_export str JSONprint(int *ret, stre
json_export str JSONstore(int *ret, str *nme, int *kind, int *string, int
*integer, int *doble, int *array, int *object, int *name);
json_export str JSONload(int *kind, int *string, int *integer, int *doble, int
*array, int *object, int *name, str *nme);
json_export str JSONdrop(int *ret, str *name);
+json_export str JSONextract(int *rkind, int *rstring, int *rinteger, int
*rdoble, int *rarray, int *robject, int *rname, int *kind, int *string, int
*integer, int *doble, int *array, int *object, int *name, int *elems, oid
*startoid);
#endif
diff --git a/monetdb5/extras/jaql/json.mal b/monetdb5/extras/jaql/json.mal
--- a/monetdb5/extras/jaql/json.mal
+++ b/monetdb5/extras/jaql/json.mal
@@ -36,3 +36,7 @@ comment "Load the JSON object name from
command drop(nme:str):void
address JSONdrop
comment "Drop the JSON object name from the database";
+
+command
extract(k:bat[:oid,:chr],s:bat[:oid,:str],i:bat[:oid,:lng],d:bat[:oid,:dbl],a:bat[:oid,:oid],o:bat[:oid,:oid],n:bat[:oid,:str],elems:bat[:oid,:oid],start:oid)(kind:bat[:oid,:chr],string:bat[:oid,:str],integer:bat[:oid,:lng],double:bat[:oid,:dbl],array:bat[:oid,:oid],object:bat[:oid,:oid],name:bat[:oid,:str])
+address JSONextract
+comment "Extract the given elems (kind id) from the given JSON object";
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
