Changeset: 5ebb590d5052 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5ebb590d5052
Modified Files:
sql/backends/monet5/vaults/fits/fits.c
Branch: Mar2018
Log Message:
Extend the FITS data vault to read thumbnails
FITS files may contain thumbnails. Store them as blobs in the monetdb
table on load.
diffs (218 lines):
diff --git a/sql/backends/monet5/vaults/fits/fits.c
b/sql/backends/monet5/vaults/fits/fits.c
--- a/sql/backends/monet5/vaults/fits/fits.c
+++ b/sql/backends/monet5/vaults/fits/fits.c
@@ -85,8 +85,11 @@ FITSinitCatalog(mvc *m)
}
static int
-fits2mtype(int t)
+fits2mtype(int t, int rep)
{
+ if (rep > 1) {
+ return TYPE_sqlblob;
+ }
switch (t) {
case TBIT:
case TLOGICAL:
@@ -121,7 +124,10 @@ fits2mtype(int t)
static int
fits2subtype(sql_subtype *tpe, int t, long rep, long wid) /* type long used by
fits library */
{
- (void)rep;
+ if (rep > 1) {
+ sql_find_subtype(tpe, "blob", (unsigned int)rep*wid, 0);
+ return 1;
+ }
switch (t) {
case TBIT:
case TLOGICAL:
@@ -203,7 +209,7 @@ str FITSexportTable(Client cntxt, MalBlk
sch = mvc_bind_schema(m, "sys");
/* First step: look if the table exists in the database. If the table
is not in the database, the export function cannot continue */
-
+
tbl = mvc_bind_table(m, sch, tname);
if (tbl == NULL) {
msg = createException (MAL, "fits.exporttable", SQLSTATE(FI000)
"Table %s is missing.\n", tname);
@@ -319,7 +325,7 @@ str FITSexportTable(Client cntxt, MalBlk
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1,
1, dimension, readboolrows, &status);
texportboolean += GDKms() - tm0;
- GDKfree(readboolrows);
+ GDKfree(readboolrows);
}
if (strcmp(columntype,"char")==0)
@@ -409,7 +415,7 @@ str FITSexportTable(Client cntxt, MalBlk
readshortrows = (short *) GDKmalloc
(sizeof(short) * optimal);
block++;
}
- }
+ }
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1,
1, dimension, readshortrows, &status);
texportshort += GDKms() - tm0;
@@ -439,11 +445,11 @@ str FITSexportTable(Client cntxt, MalBlk
readintrows = (int *) GDKmalloc
(sizeof(int) * optimal);
block++;
}
- }
+ }
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1,
dimension, readintrows, &status);
texportint += GDKms() - tm0;
- GDKfree(readintrows);
+ GDKfree(readintrows);
}
if (strcmp(columntype,"bigint")==0)
@@ -469,7 +475,7 @@ str FITSexportTable(Client cntxt, MalBlk
readlngrows = (lng *) GDKmalloc
(sizeof(lng) * optimal);
block++;
}
- }
+ }
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1,
dimension, readlngrows, &status);
texportlng += GDKms() - tm0;
@@ -499,7 +505,7 @@ str FITSexportTable(Client cntxt, MalBlk
readfloatrows = (float *) GDKmalloc
(sizeof(float) * optimal);
block++;
}
- }
+ }
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1,
1, dimension, readfloatrows, &status);
texportfloat += GDKms() - tm0;
@@ -533,12 +539,12 @@ str FITSexportTable(Client cntxt, MalBlk
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1,
1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
- GDKfree(readdoublerows);
+ GDKfree(readdoublerows);
}
}
/* print all the times that were needed to export each one of the
columns
-
+
fprintf(stderr, "\n\n");
if (texportboolean > 0) fprintf(stderr, "%d Boolean\tcolumn(s)
exported in %d ms\n", boolcols, texportboolean);
if (texportchar > 0) fprintf(stderr, "%d Char\t\tcolumn(s)
exported in %d ms\n", charcols, texportchar);
@@ -942,6 +948,7 @@ str FITSloadTable(Client cntxt, MalBlkPt
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0,
cnum);
+ // TODO: Check that the allocations succeeded
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
@@ -970,24 +977,81 @@ str FITSloadTable(Client cntxt, MalBlkPt
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
- mtype = fits2mtype(tpcode[j - 1]);
+ mtype = fits2mtype(tpcode[j - 1], rep[j - 1]);
nilptr = ATOMnilptr(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = COLnew(0, mtype, rows, TRANSIENT);
- if ( tmp == NULL){
+ if (tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
- if (mtype != TYPE_str) {
- fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, (void
*) nilptr, (void *)BUNtloc(bat_iterator(tmp), 0), &anynull, &status);
- BATsetcount(tmp, rows);
- tmp->tsorted = 0;
- tmp->trevsorted = 0;
- } else {
+ if (mtype == TYPE_sqlblob) {
+ long i;
+ unsigned long nbytes = rep[j - 1] * wid[j - 1];
+ sqlblob **v = (sqlblob **)GDKzalloc(sizeof(sqlblob *) *
rows);
+ char *buff = (char *)GDKzalloc(nbytes);
+ size_t k = 0;
+
+ mtype = fits2mtype(tpcode[j - 1], 1);
+ nilptr = ATOMnilptr(mtype);
+
+ if (v == NULL) {
+ GDKfree(tpcode);
+ GDKfree(rep);
+ GDKfree(wid);
+ GDKfree(cname);
+ throw(MAL,"fits.load", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
+ }
+
+ for(i = 0; i < rows; i++) {
+ v[i] = (sqlblob *)GDKmalloc(offsetof(sqlblob,
data) + nbytes);
+ if (v[i] == NULL) {
+ GDKfree(tpcode);
+ GDKfree(rep);
+ GDKfree(wid);
+ GDKfree(cname);
+ long k = 0;
+ for (k = 0; k < i; k++) {
+ GDKfree(v[k]);
+ }
+ GDKfree(v);
+ GDKfree(buff);
+ throw(MAL,"fits.load", SQLSTATE(HY001)
MAL_MALLOC_FAIL);
+ }
+ fits_read_col(fptr, tpcode[j - 1], j, i + 1, 1,
rep[j - 1], (void *)nilptr,
+ (void *)buff, &anynull, &status);
+ for (k = 0; k < nbytes; k++) {
+ v[i]->data[k] = *(buff + k);
+ k++;
+ }
+ v[i]->nitems = nbytes;
+ if (BUNappend(tmp, v[i], TRUE) != GDK_SUCCEED) {
+ BBPreclaim(tmp);
+ msg = createException(MAL,
"fits.loadtable", SQLSTATE(HY001) MAL_MALLOC_FAIL);
+ GDKfree(tpcode);
+ GDKfree(rep);
+ GDKfree(wid);
+ GDKfree(cname);
+ for (i = 0; i < rows; i++) {
+ GDKfree(v[i]);
+ }
+ GDKfree(v);
+ GDKfree(buff);
+ return msg;
+ }
+ }
+
+ for(i = 0; i < rows; i++) {
+ GDKfree(v[i]);
+ }
+ GDKfree(v);
+ GDKfree(buff);
+ }
+ else if (mtype == TYPE_str) {
/* char *v = GDKzalloc(wid[j-1]);*/
/* type long demanded by "rows", i.e., by fits library
*/
long bsize = 50, batch = bsize, k, i;
@@ -1014,6 +1078,12 @@ str FITSloadTable(Client cntxt, MalBlkPt
GDKfree(v);
fprintf(stderr,"#String column load %d ms, BUNappend %d
ms\n", tloadtm, tattachtm);
}
+ else {
+ fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, (void
*) nilptr, (void *)BUNtloc(bat_iterator(tmp), 0), &anynull, &status);
+ BATsetcount(tmp, rows);
+ tmp->tsorted = 0;
+ tmp->trevsorted = 0;
+ }
if (status) {
char buf[FLEN_ERRMSG + 1];
@@ -1041,4 +1111,3 @@ str FITSloadTable(Client cntxt, MalBlkPt
fits_close_file(fptr, &status);
return msg;
}
-
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
