Changeset: 3df92304df1c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3df92304df1c
Modified Files:
monetdb5/extras/dvf/dvf.c
monetdb5/extras/dvf/opt_dvf.c
sql/backends/monet5/bam/Tests/benchmark1.sql
sql/backends/monet5/bam/bam_loader.c
sql/backends/monet5/bam/bam_schema_1.sql
Branch: DVframework_bam
Log Message:
Did some work on adapting dvf.c and opt_dvf.c for working with the BAM schema
(these files will have to be made more generic later on).
Worked some on benchmark1.sql.
Did some bugfixing in bam_loader and made a minor change to the schema.
diffs (truncated from 527 to 300 lines):
diff --git a/monetdb5/extras/dvf/dvf.c b/monetdb5/extras/dvf/dvf.c
--- a/monetdb5/extras/dvf/dvf.c
+++ b/monetdb5/extras/dvf/dvf.c
@@ -10,7 +10,7 @@
int get_column_type(str schema_name, str table_name, int column_num)
{
- if(strcmp(schema_name, "mseed") != 0 || strcmp(table_name, "data") != 0)
+ /*if(strcmp(schema_name, "mseed") != 0 || strcmp(table_name, "data") !=
0)
return -1;
switch(column_num)
@@ -25,12 +25,37 @@ int get_column_type(str schema_name, str
return TYPE_int;
default:
return -1;
+ }*/
+
+ if(strcmp(schema_name, "bam") != 0 || strcmp(table_name, "alignments")
!= 0)
+ return -1;
+
+ switch(column_num)
+ {
+ case 0:
+ return TYPE_lng;
+ case 1:
+ case 3:
+ case 6:
+ return TYPE_sht;
+ case 2:
+ case 4:
+ case 7:
+ case 8:
+ case 11:
+ case 12:
+ return TYPE_str;
+ case 5:
+ case 9:
+ case 10:
+ return TYPE_int;
+ default: return -1;
}
}
int get_column_num(str schema_name, str table_name, str column_name)
{
- if(strcmp(schema_name, "mseed") != 0 || strcmp(table_name, "data") != 0)
+ /*if(strcmp(schema_name, "mseed") != 0 || strcmp(table_name, "data") !=
0)
return -1;
if(strcmp(column_name, "file_location") == 0)
@@ -42,12 +67,44 @@ int get_column_num(str schema_name, str
else if(strcmp(column_name, "sample_value") == 0)
return 3;
else
+ return -1;*/
+
+ if(strcmp(schema_name, "bam") != 0 || strcmp(table_name, "alignments")
!= 0)
+ return -1;
+
+ if(strcmp(column_name, "virtual_offset") == 0)
+ return 0;
+ else if(strcmp(column_name, "file_id") == 0)
+ return 1;
+ else if(strcmp(column_name, "qname") == 0)
+ return 2;
+ else if(strcmp(column_name, "flag") == 0)
+ return 3;
+ else if(strcmp(column_name, "rname") == 0)
+ return 4;
+ else if(strcmp(column_name, "pos") == 0)
+ return 5;
+ else if(strcmp(column_name, "mapq") == 0)
+ return 6;
+ else if(strcmp(column_name, "cigar") == 0)
+ return 7;
+ else if(strcmp(column_name, "rnext") == 0)
+ return 8;
+ else if(strcmp(column_name, "pnext") == 0)
+ return 9;
+ else if(strcmp(column_name, "tlen") == 0)
+ return 10;
+ else if(strcmp(column_name, "seq") == 0)
+ return 11;
+ else if(strcmp(column_name, "qual") == 0)
+ return 12;
+ else
return -1;
}
str plan_modifier(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
- str data_table_identifier = "data";
+ str data_table_identifier = "alignments";
str mountRef = putName("mount", 5);
str miniseedRef = putName("miniseed", 8);
str dvfRef = putName("dvf", 3);
diff --git a/monetdb5/extras/dvf/opt_dvf.c b/monetdb5/extras/dvf/opt_dvf.c
--- a/monetdb5/extras/dvf/opt_dvf.c
+++ b/monetdb5/extras/dvf/opt_dvf.c
@@ -101,9 +101,9 @@ OPTdvfImplementation(Client cntxt, MalBl
{
//TODO: Replace these with a proper (global) constants
str sys_schema_name = "sys";
- str schema_name = "mseed";
- str data_table_identifier = "data";
- str file_location_identifier = "file_location";
+ str schema_name = "bam";
+ str data_table_identifier = "alignments";
+ str file_location_identifier = "file_id";
str mountRef = putName("mount", 5);
str miniseedRef = putName("miniseed", 8);
str dvfRef = putName("dvf", 3);
diff --git a/sql/backends/monet5/bam/Tests/benchmark1.sql
b/sql/backends/monet5/bam/Tests/benchmark1.sql
--- a/sql/backends/monet5/bam/Tests/benchmark1.sql
+++ b/sql/backends/monet5/bam/Tests/benchmark1.sql
@@ -3,7 +3,7 @@
--------------------------------------------------------------------------------------------------------------------------------------
SELECT *
-FROM alignments
+FROM bam.alignments
WHERE file_id = 1
ORDER BY qname;
@@ -21,7 +21,7 @@ ORDER BY qname;
--------------------------------------------------------------------------------------------------------------------------------------
SELECT *
-FROM alignments
+FROM bam.alignments
WHERE file_id = 1
ORDER BY rname, pos;
@@ -39,7 +39,7 @@ ORDER BY rname, pos;
--------------------------------------------------------------------------------------------------------------------------------------
SELECT *
-FROM alignments
+FROM bam.alignments
WHERE file_id = 1
AND (rname <= 'chr22' OR rname >= 'chr11')
AND (pos <= 2000000 OR pos >= 1000000);
@@ -58,6 +58,6 @@ WHERE file_id = 1
--------------------------------------------------------------------------------------------------------------------------------------
SELECT *
-FROM alignments
+FROM bam.alignments
WHERE file_id = 1
AND qname = 'sim_22_1_a'
diff --git a/sql/backends/monet5/bam/bam_loader.c
b/sql/backends/monet5/bam/bam_loader.c
--- a/sql/backends/monet5/bam/bam_loader.c
+++ b/sql/backends/monet5/bam/bam_loader.c
@@ -24,15 +24,6 @@
continue; \
}
-
-#define _append_option_to_bat_cond_flt(ret_tc, opt, cmp, table, col,
appendErr, flag, f) \
- if(strcmp((opt)->tag, (cmp)) == 0) \
- { \
- (f) = strtof((opt)->value, NULL); \
- _append_option_to_bat((ret_tc), (ptr)&f, (table), (col), (appendErr),
(flag)); \
- continue; \
- } \
-
#define _append_option_to_bat(ret_tc, value, table, col, appendErr, flag) \
if(BUNappend((ret_tc)->tables_columns[(table)].column_bats[(col)],
(value), TRUE) == NULL) \
*appendErr = TRUE; \
@@ -93,13 +84,13 @@ sht _num_col_pg = 6;
sht _num_col_alignments_extra = 5;
str _coln_files[] = {"file_id", "file_location", "format_version",
"sorting_order", "comments"};
-int _colt_files[] = {TYPE_sht , TYPE_str , TYPE_flt , TYPE_str
, TYPE_str };
+int _colt_files[] = {TYPE_sht , TYPE_str , TYPE_str , TYPE_str
, TYPE_str };
str _coln_sq[] = {"sn" , "file_id", "ln" , "as" , "m5" , "sp"
, "ur" };
int _colt_sq[] = {TYPE_str, TYPE_sht , TYPE_int, TYPE_int, TYPE_str,
TYPE_str, TYPE_str};
str _coln_rg[] = {"id" , "file_id", "cn" , "ds" , "dt" , "fo"
, "ks" , "lb" , "pg" , "pi" , "pl" , "pu" , "sm" };
-int _colt_rg[] = {TYPE_str, TYPE_sht , TYPE_str, TYPE_str, TYPE_int,
TYPE_str, TYPE_str, TYPE_str, TYPE_str, TYPE_int, TYPE_str, TYPE_str, TYPE_str};
+int _colt_rg[] = {TYPE_str, TYPE_sht , TYPE_str, TYPE_str, TYPE_lng,
TYPE_str, TYPE_str, TYPE_str, TYPE_str, TYPE_int, TYPE_str, TYPE_str, TYPE_str};
str _coln_pg[] = {"id" , "file_id", "pn" , "cl" , "pp" , "vn"
};
int _colt_pg[] = {TYPE_str, TYPE_sht , TYPE_str, TYPE_str, TYPE_str, TYPE_str
};
@@ -111,6 +102,8 @@ int _colt_alignments_extra[] = {TYPE_st
/* File format specific functions */
+static lng _get_line_num(str filename);
+static lng _get_file_paths(str repo_path, str** ret_file_paths);
static str _next_file_id(Client cntxt, MalBlkPtr mb, sht *next_file_id);
static str _init_temp_container(_temp_container *ret_tc, sht dbschema);
static str _init_temp_container_simple(_temp_container *ret_tc);
@@ -136,40 +129,167 @@ bam_loader(Client cntxt, MalBlkPtr mb, M
/* TODO: This function is now called for BAM files, eventually there will
have to be one generic */
/* function that e.g. receives the file format for which a DV should be
initialized as an argument */
- str *repo_path = (str*) getArgReference(stk,pci,pci->retc); /* arg 1:
repo_path, used as filepath right now */
+ str *repo_path = (str*) getArgReference(stk,pci,pci->retc); /* arg 1:
repo_path */
sht dbschema = *(sht*) getArgReference(stk,pci,pci->retc+1); /* arg 2:
schema to use, see bam_loader.h for possible options */
- sht num_threads = *(sht*) getArgReference(stk,pci,pci->retc+2); /* arg
3: 1: no threads, >1: multi-threaded */
+ sht nr_threads = *(sht*) getArgReference(stk,pci,pci->retc+2); /* arg
3: 1: no threads, >1: multi-threaded */
- sht next_file_id = 0;
+ str *file_paths;
+ int nr_files, k;
- str err1 = NULL, err2 = NULL;
- _temp_container *tc = (_temp_container
*)GDKmalloc(sizeof(_temp_container));
+ /* Get the file list */
+ if((nr_files = _get_file_paths(*repo_path, &file_paths)) < 0)
+ throw(MAL, "bam_loader", "Couldn't retrieve file list from file
'%s'\n", *repo_path);
- if(tc == NULL)
- throw(MAL, "bamloader", MAL_MALLOC_FAIL);
- else if((err1 = _next_file_id(cntxt, mb, &next_file_id)) != MAL_SUCCEED)
- err2 = "Error while retrieving next file_id: %s\n";
- else if((err1 = _init_temp_container(tc, dbschema)) != MAL_SUCCEED)
- err2 = "Error while creating _temp_container: %s\n";
- else if((err1 = _loadfile(*repo_path, tc, dbschema, next_file_id)) !=
MAL_SUCCEED)
+ for(k=0; k<nr_files; k++)
{
+ str file_path = file_paths[k];
+ sht next_file_id = 0;
+ str err1 = NULL, err2 = NULL;
+ _temp_container *tc = (_temp_container
*)GDKmalloc(sizeof(_temp_container));
+
+ if(tc == NULL)
+ throw(MAL, "bam_loader", MAL_MALLOC_FAIL);
+
+ if((err1 = _next_file_id(cntxt, mb, &next_file_id)) != MAL_SUCCEED)
+ err2 = "Error while retrieving next file_id: %s\n";
+ else if((err1 = _init_temp_container(tc, dbschema)) != MAL_SUCCEED)
+ err2 = "Error while creating _temp_container: %s\n";
+ else if((err1 = _loadfile(file_path, tc, dbschema, next_file_id)) !=
MAL_SUCCEED)
+ {
+ _free_temp_container(tc);
+ err2 = "Error while loading BAM file: %s\n";
+ }
+ else if((err1 = _insert_into_vault(cntxt, tc)) != MAL_SUCCEED)
+ {
+ _free_temp_container(tc);
+ err2 = "Error inserting data into database: %s\n";
+ }
+
+ if(err2 != NULL)
+ {
+ GDKfree(file_paths);
+ throw(MAL, "bam_loader", err2, err1);
+ }
_free_temp_container(tc);
- err2 = "Error while loading BAM file: %s\n";
- }
- else if((err1 = _insert_into_vault(cntxt, tc)) != MAL_SUCCEED)
- {
- _free_temp_container(tc);
- err2 = "Error inserting data into database: %s\n";
}
- if(err2 != NULL)
- throw(MAL, "bamloader", err2, err1);
+ GDKfree(file_paths);
- (void)num_threads;
+ (void)nr_threads;
return MAL_SUCCEED;
}
+
+/*
+ * Note: Copied directly from registrar.c
+
+ * returns number of lines in a file.
+ *
+ * WARNING: always counts EOF as a line. So proper return is taken if the file
does
+ * not have a newline at the end.
+ */
+static lng
+_get_line_num(str filename)
+{
+ FILE *f;
+ char c;
+ lng lines = 0;
+
+ f = fopen(filename, "r");
+
+ if(f == NULL)
+ return 0;
+
+ while((c = fgetc(f)) != EOF)
+ if(c == '\n')
+ lines++;
+
+ fclose(f);
+
+/* if(c != '\n') */
+/* lines++; */
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
