Changeset: 3fa44b4916b4 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3fa44b4916b4
Modified Files:
monetdb5/extras/rdf/rdf.h
monetdb5/extras/rdf/rdfschema.c
monetdb5/extras/rdf/rdfschema.mal
sql/backends/monet5/sql.mx
Branch: rdf
Log Message:
Check the duplication while creating triples table.
- By now, only check the duplication for SPO table.
Since the BATs (S,P,O) are sorted (from post_processing fucntion),
the duplication is checked by comparing the triples of two to-be-inserted
consecutive rows.
diffs (truncated from 407 to 300 lines):
diff --git a/monetdb5/extras/rdf/rdf.h b/monetdb5/extras/rdf/rdf.h
--- a/monetdb5/extras/rdf/rdf.h
+++ b/monetdb5/extras/rdf/rdf.h
@@ -58,7 +58,10 @@ RDFSchemaExplore(int *ret, str *tbname,
rdf_export str
RDFextractCS(int *ret, bat *sbatid, bat *pbatid);
+rdf_export str
+RDFextractPfromPSO(int *ret, bat *pbatid, bat *sbatid);
+#define IS_DUPLICATE_FREE 0 /* 0: Duplications have not been removed,
otherwise 1 */
#define TRIPLE_STORE 1
#define MLA_STORE 2
diff --git a/monetdb5/extras/rdf/rdfschema.c b/monetdb5/extras/rdf/rdfschema.c
--- a/monetdb5/extras/rdf/rdfschema.c
+++ b/monetdb5/extras/rdf/rdfschema.c
@@ -73,6 +73,31 @@ static void putCStoHash(map_t csmap, int
}
+static void putPtoHash(map_t pmap, int value, oid *poid){
+ oid *getPoid;
+ oid *putPoid;
+ int err;
+ int* pkey;
+ int freq;
+
+ pkey = (int*) malloc(sizeof(int));
+
+ *pkey = value;
+
+ if (hashmap_get(pmap, pkey, 1,(void**)(&getPoid),1, &freq) != MAP_OK){
+ putPoid = malloc(sizeof(oid));
+ *putPoid = *poid;
+
+ err = hashmap_put(pmap, pkey, 1, putPoid);
+ assert(err == MAP_OK);
+
+ (*poid)++;
+ }
+ else{
+ free(pkey);
+ }
+}
+
static void getTopFreqCSs(map_t csmap, int threshold){
int count;
hashmap_map* m;
@@ -148,6 +173,7 @@ static void getStatisticCSsBySupports(ma
free(statCS);
}
+/* Extract CS from SPO triples table */
str
RDFextractCS(int *ret, bat *sbatid, bat *pbatid){
BUN p, q;
@@ -158,7 +184,7 @@ RDFextractCS(int *ret, bat *sbatid, bat
oid curP; /* current Property oid */
oid CSoid = 0; /* Characteristic set oid */
int numP; /* Number of properties for current S */
- map_t csMap;
+ map_t csMap;
int* buff;
int INIT_PROPERTY_NUM = 50000;
int maxNumProp = 0;
@@ -233,3 +259,63 @@ RDFextractCS(int *ret, bat *sbatid, bat
*ret = 1;
return MAL_SUCCEED;
}
+
+
+/* Extract Properties and their supports from PSO table */
+str
+RDFextractPfromPSO(int *ret, bat *sbatid, bat *pbatid){
+ BUN p, q;
+ BAT *sbat = NULL, *pbat = NULL;
+ BATiter si, pi; /*iterator for BAT of s,p columns in spo table
*/
+ oid *bt, *sbt;
+ oid curS; /* current Subject oid */
+ oid curP; /* current Property oid */
+ map_t pMap;
+ int supportP; /* Support value for P */
+ oid Poid = 0; /* Characteristic set oid */
+
+ if ((sbat = BATdescriptor(*sbatid)) == NULL) {
+ throw(MAL, "rdf.RDFextractCS", RUNTIME_OBJECT_MISSING);
+ }
+ if ((pbat = BATdescriptor(*pbatid)) == NULL) {
+ throw(MAL, "rdf.RDFextractCS", RUNTIME_OBJECT_MISSING);
+ }
+
+ si = bat_iterator(sbat);
+ pi = bat_iterator(pbat);
+
+ /* Init a hashmap */
+ pMap = hashmap_new();
+ curP = 0;
+ supportP = 0;
+
+ BATloop(pbat, p, q){
+ bt = (oid *) BUNtloc(pi, p);
+ if (*bt != curP){
+ if (p != 0){ /* Not the first S */
+ putPtoHash(pMap, *bt, &Poid);
+ }
+ curP = *bt;
+ curS = 0;
+ }
+
+ sbt = (oid *) BUNtloc(si, p);
+
+ if (curS != *sbt){
+ supportP++;
+ curS = *sbt;
+ }
+ }
+
+ /*put the last P */
+ putPtoHash(pMap, *bt, &Poid);
+
+
+ BBPreclaim(sbat);
+ BBPreclaim(pbat);
+
+ hashmap_free(pMap);
+
+ *ret = 1;
+ return MAL_SUCCEED;
+}
diff --git a/monetdb5/extras/rdf/rdfschema.mal
b/monetdb5/extras/rdf/rdfschema.mal
--- a/monetdb5/extras/rdf/rdfschema.mal
+++ b/monetdb5/extras/rdf/rdfschema.mal
@@ -24,5 +24,8 @@ comment "Explore the schema information
command rdfextractCS( sbat:bat[:any_1,:oid], pbat:bat[:any_2,:oid] ) :void
address RDFextractCS
-comment "Extract Characteristic sets from SPO tables";
+comment "Extract Characteristic sets from SPO table";
+command rdfextractPfromPSO(pbat:bat[:any_1,:oid], sbat:bat[:any_2,:oid]):void
+address RDFextractPfromPSO
+comment "Extract Properties and their supports from PSO table";
diff --git a/sql/backends/monet5/sql.mx b/sql/backends/monet5/sql.mx
--- a/sql/backends/monet5/sql.mx
+++ b/sql/backends/monet5/sql.mx
@@ -7179,6 +7179,12 @@ SQLrdfShred(Client cntxt, MalBlkPtr mb,
int id = 0;
oid rid = oid_nil;
str msg;
+#if IS_DUPLICATE_FREE == 0
+ BATiter si, pi, oi;
+ BUN pb, qb;
+ oid *sbt, *pbt, *obt;
+ oid curS = 0, curP = 0, curO = 0;
+#endif /* IS_DUPLICATE_FREE */
rethrow("sql.rdfShred", msg, getSQLContext(cntxt, mb, &m, NULL));
@@ -7276,78 +7282,175 @@ SQLrdfShred(Client cntxt, MalBlkPtr mb,
BBPunfix(s->batCacheid);
#if STORE == TRIPLE_STORE
- s = b[S_sort];
- p = b[P_PO];
- o = b[O_PO];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, spo_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, spo_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, spo_tbl, "object"),
- o, TYPE_bat);
- s = b[S_sort];
- p = b[P_OP];
- o = b[O_OP];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, sop_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, sop_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, sop_tbl, "object"),
- o, TYPE_bat);
- s = b[S_SO];
- p = b[P_sort];
- o = b[O_SO];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pso_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pso_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pso_tbl, "object"),
- o, TYPE_bat);
- s = b[S_OS];
- p = b[P_sort];
- o = b[O_OS];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pos_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pos_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, pos_tbl, "object"),
- o, TYPE_bat);
- s = b[S_SP];
- p = b[P_SP];
- o = b[O_sort];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, osp_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, osp_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, osp_tbl, "object"),
- o, TYPE_bat);
- s = b[S_PS];
- p = b[P_PS];
- o = b[O_sort];
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, ops_tbl, "subject"),
- s, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, ops_tbl, "property"),
- p, TYPE_bat);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, ops_tbl, "object"),
- o, TYPE_bat);
+ #if IS_DUPLICATE_FREE == 0
+
+ s = b[S_sort];
+ p = b[P_PO];
+ o = b[O_PO];
+ si = bat_iterator(s);
+ pi = bat_iterator(p);
+ oi = bat_iterator(o);
+
+ BATloop(s, pb, qb){
+ sbt = (oid *) BUNtloc(si, pb);
+ pbt = (oid *) BUNtloc(pi, pb);
+ obt = (oid *) BUNtloc(oi, pb);
+
+ if (*sbt != curS || *pbt != curP || *obt != curO){
+
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m,
spo_tbl, "subject"),
+ sbt, TYPE_int);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m,
spo_tbl, "property"),
+ pbt, TYPE_int);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m,
spo_tbl, "object"),
+ obt, TYPE_int);
+ /* Update current value */
+ curS = *sbt;
+ curP = *pbt;
+ curO = *obt;
+ }
+ }
+ s = b[S_sort];
+ p = b[P_OP];
+ o = b[O_OP];
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, sop_tbl, "subject"),
+ s, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, sop_tbl, "property"),
+ p, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, sop_tbl, "object"),
+ o, TYPE_bat);
+ s = b[S_SO];
+ p = b[P_sort];
+ o = b[O_SO];
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, pso_tbl, "subject"),
+ s, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, pso_tbl, "property"),
+ p, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, pso_tbl, "object"),
+ o, TYPE_bat);
+ s = b[S_OS];
+ p = b[P_sort];
+ o = b[O_OS];
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, pos_tbl, "subject"),
+ s, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
+ mvc_bind_column(m, pos_tbl, "property"),
+ p, TYPE_bat);
+ store_funcs.append_col(m->session->tr,
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
