Changeset: eb9a9582ba77 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=eb9a9582ba77
Modified Files:
sql/backends/monet5/sql.c
sql/common/sql_types.c
sql/server/rel_updates.c
sql/server/sql_parser.y
sql/server/sql_scan.c
Branch: fixed-width-format
Log Message:
COPY INTO table FWF (1,2,3); now works
diffs (267 lines):
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -3410,10 +3410,16 @@ mvc_import_table_wrap(Client cntxt, MalB
lng *offset = getArgReference_lng(stk, pci, pci->retc + 7);
int *locked = getArgReference_int(stk, pci, pci->retc + 8);
int *besteffort = getArgReference_int(stk, pci, pci->retc + 9);
+ char *fixed_widths = NULL;
str msg = MAL_SUCCEED;
bstream *s = NULL;
stream *ss;
+ if (pci->argc - pci->retc > 10) {
+ fixed_widths = *getArgReference_str(stk, pci, pci->retc + 10);
+
+ }
+
(void) mb; /* NOT USED */
if ((msg = checkSQLContext(cntxt)) != NULL)
return msg;
@@ -3483,14 +3489,29 @@ mvc_import_table_wrap(Client cntxt, MalB
return msg;
}
GDKfree(fn);
-
- // FIXME
- if (GDKgetenv_isyes("testfwf")) {
- fprintf(stderr, "### FWF IMPORT ON %s ###\n", *fname);
- size_t *widths = malloc(sizeof(size_t)*400);
- size_t i = 0;
- widths[i++] = 2; widths[i++] = 2; widths[i++] = 9;
widths[i++] = 12; widths[i++] = 8; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 8;
widths[i++] = 8; widths[i++] = 30; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 9; widths[i++] = 4; widths[i++] = 4; widths[i++] = 2;
widths[i++] = 8; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 9; widths[i++] = 9; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 9; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; wi
dths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 4;
widths[i++] = 2; widths[i++] = 8; widths[i++] = 2; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i+
+] = 2; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] =
3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9;
widths[i++] = 9; widths[i++] = 8; widths[i++] = 8; widths[i++] = 8;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; widths[i++] = 9;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 9; widths[i++] = 9;
widths[i++] = 9; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 9; widths[i++] = 9; width
s[i++] = 9; widths[i++] = 4; widths[i++] = 4; widths[i++] = 4; widths[i++]
= 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++]
= 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; w
idths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 12; widths[i++] = 12; widths[i++] = 12; widths[i++] = 8;
widths[i++] = 8; widths[i++] = 8; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 3; widths[i++] = 3; widths[i++] = 3;
widths[i++] = 3; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 2;
widths[i++] = 2; widths[i++] = 2; widths[i++] = 2; widths[i++] = 40;
- ss = stream_fwf_create(ss, 376, widths, ' ');
+ if (fixed_widths) {
+ size_t ncol = 0, current_width_entry = 0, i;
+ size_t *widths;
+ char* val_start = fixed_widths;
+ size_t width_len = strlen(fixed_widths);
+ for (i = 0; i < width_len; i++) {
+ if (fixed_widths[i] == '|') {
+ ncol++;
+ }
+ }
+ widths = malloc(sizeof(size_t) * ncol);
+ if (!widths) {
+ // TODO: free other stuff
+ throw(MAL, "sql.copy_from", MAL_MALLOC_FAIL);
+ }
+ for (i = 0; i < width_len; i++) {
+ if (fixed_widths[i] == '|') {
+ fixed_widths[i] = '\0';
+ widths[current_width_entry++] =
(size_t) atoll(val_start);
+ val_start = fixed_widths + i + 1;
+ }
+ }
+ ss = stream_fwf_create(ss, ncol, widths, ' ');
}
#if SIZEOF_VOID_P == 4
s = bstream_create(ss, 0x20000);
diff --git a/sql/common/sql_types.c b/sql/common/sql_types.c
--- a/sql/common/sql_types.c
+++ b/sql/common/sql_types.c
@@ -1760,9 +1760,9 @@ sqltypeinit( sql_allocator *sa)
sres, FALSE, F_FUNC, SCALE_FIX);
}
sres = create_arg(sa, NULL, sql_create_subtype(sa, TABLE, 0, 0),
ARG_OUT);
- /* copyfrom fname (arg 9) */
+ /* copyfrom fname (arg 10) */
f=sql_create_func_(sa, "copyfrom", "sql", "copy_from",
- list_append( list_append( list_append( list_append(
list_append(list_append (list_append (list_append(list_append(sa_list(sa),
+ list_append( list_append( list_append( list_append(
list_append(list_append (list_append
(list_append(list_append(list_append(sa_list(sa),
create_arg(sa, NULL, sql_create_subtype(sa, STR, 0, 0),
ARG_IN)),
create_arg(sa, NULL, sql_create_subtype(sa, STR, 0, 0),
ARG_IN)),
create_arg(sa, NULL, sql_create_subtype(sa, STR, 0, 0),
ARG_IN)),
@@ -1771,7 +1771,8 @@ sqltypeinit( sql_allocator *sa)
create_arg(sa, NULL, sql_create_subtype(sa, STR, 0, 0),
ARG_IN)),
create_arg(sa, NULL, sql_create_subtype(sa, LNG, 0, 0),
ARG_IN)),
create_arg(sa, NULL, sql_create_subtype(sa, LNG, 0, 0),
ARG_IN)),
- create_arg(sa, NULL, sql_create_subtype(sa, INT, 0, 0),
ARG_IN)), sres, FALSE, F_UNION, SCALE_FIX);
+ create_arg(sa, NULL, sql_create_subtype(sa, INT, 0, 0),
ARG_IN)),
+ create_arg(sa, NULL, sql_create_subtype(sa, STR, 0, 0),
ARG_IN)), sres, FALSE, F_UNION, SCALE_FIX);
f->varres = 1;
/* bincopyfrom */
diff --git a/sql/server/rel_updates.c b/sql/server/rel_updates.c
--- a/sql/server/rel_updates.c
+++ b/sql/server/rel_updates.c
@@ -1136,7 +1136,7 @@ table_column_types(sql_allocator *sa, sq
}
static sql_rel *
-rel_import(mvc *sql, sql_table *t, char *tsep, char *rsep, char *ssep, char
*ns, char *filename, lng nr, lng offset, int locked, int best_effort)
+rel_import(mvc *sql, sql_table *t, char *tsep, char *rsep, char *ssep, char
*ns, char *filename, lng nr, lng offset, int locked, int best_effort, dlist
*fwf_widths)
{
sql_rel *res;
list *exps, *args;
@@ -1144,7 +1144,8 @@ rel_import(mvc *sql, sql_table *t, char
sql_subtype tpe;
sql_exp *import;
sql_schema *sys = mvc_bind_schema(sql, "sys");
- sql_subfunc *f = sql_find_func(sql->sa, sys, "copyfrom", 9, F_UNION,
NULL);
+ sql_subfunc *f = sql_find_func(sql->sa, sys, "copyfrom", 10, F_UNION,
NULL);
+ char* fwf_string = NULL;
if (!f) /* we do expect copyfrom to be there */
return NULL;
@@ -1157,16 +1158,37 @@ rel_import(mvc *sql, sql_table *t, char
exp_atom_str(sql->sa, ssep, &tpe)),
exp_atom_str(sql->sa, ns, &tpe));
+ if (fwf_widths && dlist_length(fwf_widths) > 0) {
+ dnode *dn;
+ int ncol = 0;
+ fwf_string = GDKmalloc(20 * dlist_length(fwf_widths) + 1); // a
64 bit int needs 19 characters in decimal representation plus the separator
+ if (!fwf_string) {
+ return NULL;
+ }
+ char* fwf_string_cur = fwf_string;
+ for (dn = fwf_widths->h; dn; dn = dn->next) {
+ fwf_string_cur += sprintf(fwf_string_cur, LLFMT"|",
dn->data.l_val);
+ ncol++;
+ }
+ if(list_length(f->res) != ncol) {
+ (void) sql_error(sql, 02, "3F000!COPY INTO: fixed width
import for %d columns but %d widths given.", list_length(f->res), ncol);
+ return NULL;
+ }
+ *fwf_string_cur = '\0';
+ }
+
append( args, exp_atom_str(sql->sa, filename, &tpe));
import = exp_op(sql->sa,
append(
append(
append(
- append( args,
- exp_atom_lng(sql->sa, nr)),
- exp_atom_lng(sql->sa, offset)),
- exp_atom_int(sql->sa, locked)),
- exp_atom_int(sql->sa, best_effort)),
f);
+ append(
+ append( args,
+ exp_atom_lng(sql->sa, nr)),
+ exp_atom_lng(sql->sa, offset)),
+ exp_atom_int(sql->sa, locked)),
+ exp_atom_int(sql->sa,
best_effort)),
+ exp_atom_str(sql->sa,
fwf_string, &tpe)), f);
exps = new_exp_list(sql->sa);
for (n = t->columns.set->h; n; n = n->next) {
@@ -1179,7 +1201,7 @@ rel_import(mvc *sql, sql_table *t, char
}
static sql_rel *
-copyfrom(mvc *sql, dlist *qname, dlist *columns, dlist *files, dlist *headers,
dlist *seps, dlist *nr_offset, str null_string, int locked, int best_effort,
int constraint)
+copyfrom(mvc *sql, dlist *qname, dlist *columns, dlist *files, dlist *headers,
dlist *seps, dlist *nr_offset, str null_string, int locked, int best_effort,
int constraint, dlist *fwf_widths)
{
sql_rel *rel = NULL;
char *sname = qname_schema(qname);
@@ -1194,7 +1216,6 @@ copyfrom(mvc *sql, dlist *qname, dlist *
lng offset = (nr_offset)?nr_offset->h->next->data.l_val:0;
list *collist;
int reorder = 0;
-
assert(!nr_offset || nr_offset->h->type == type_lng);
assert(!nr_offset || nr_offset->h->next->type == type_lng);
if (sname && !(s=mvc_bind_schema(sql, sname))) {
@@ -1296,7 +1317,7 @@ copyfrom(mvc *sql, dlist *qname, dlist *
return sql_error(sql, 02, "COPY INTO: filename
must "
"have absolute path: %s",
fname);
- nrel = rel_import(sql, nt, tsep, rsep, ssep, ns, fname,
nr, offset, locked, best_effort);
+ nrel = rel_import(sql, nt, tsep, rsep, ssep, ns, fname,
nr, offset, locked, best_effort, fwf_widths);
if (!rel)
rel = nrel;
@@ -1306,7 +1327,7 @@ copyfrom(mvc *sql, dlist *qname, dlist *
return rel;
}
} else {
- rel = rel_import(sql, nt, tsep, rsep, ssep, ns, NULL, nr,
offset, locked, best_effort);
+ rel = rel_import(sql, nt, tsep, rsep, ssep, ns, NULL, nr,
offset, locked, best_effort, NULL);
}
if (headers) {
dnode *n;
@@ -1581,7 +1602,8 @@ rel_updates(mvc *sql, symbol *s)
l->h->next->next->next->next->next->next->data.sval,
l->h->next->next->next->next->next->next->next->data.i_val,
l->h->next->next->next->next->next->next->next->next->data.i_val,
-
l->h->next->next->next->next->next->next->next->next->next->data.i_val);
+
l->h->next->next->next->next->next->next->next->next->next->data.i_val,
+
l->h->next->next->next->next->next->next->next->next->next->next->data.lval);
sql->type = Q_UPDATE;
}
break;
diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -342,6 +342,8 @@ int yydebug=1;
assignment_commalist
opt_column_list
column_commalist_parens
+ opt_fwf_widths
+ fwf_widthlist
opt_header_list
header_list
header
@@ -599,7 +601,7 @@ SQLCODE SQLERROR UNDER WHENEVER
%token CASE WHEN THEN ELSE NULLIF COALESCE IF ELSEIF WHILE DO
%token ATOMIC BEGIN END
-%token COPY RECORDS DELIMITERS STDIN STDOUT
+%token COPY RECORDS DELIMITERS STDIN STDOUT FWF
%token INDEX
%token AS TRIGGER OF BEFORE AFTER ROW STATEMENT sqlNEW OLD EACH REFERENCING
@@ -2481,7 +2483,7 @@ opt_to_savepoint:
;
copyfrom_stmt:
- COPY opt_nr INTO qname opt_column_list FROM string_commalist
opt_header_list opt_seps opt_null_string opt_locked opt_best_effort
opt_constraint
+ COPY opt_nr INTO qname opt_column_list FROM string_commalist
opt_header_list opt_seps opt_null_string opt_locked opt_best_effort
opt_constraint opt_fwf_widths
{ dlist *l = L();
append_list(l, $4);
append_list(l, $5);
@@ -2493,6 +2495,7 @@ copyfrom_stmt:
append_int(l, $11);
append_int(l, $12);
append_int(l, $13);
+ append_list(l, $14);
$$ = _symbol_create_list( SQL_COPYFROM, l ); }
| COPY opt_nr INTO qname opt_column_list FROM STDIN opt_header_list
opt_seps opt_null_string opt_locked opt_best_effort opt_constraint
{ dlist *l = L();
@@ -2532,7 +2535,22 @@ copyfrom_stmt:
append_string(l, $6);
$$ = _symbol_create_list( SQL_COPYTO, l ); }
;
-
+
+
+
+opt_fwf_widths:
+ /* empty */ { $$ = NULL; }
+| FWF '(' fwf_widthlist ')' { $$ = $3; }
+
+ ;
+
+ fwf_widthlist:
+ poslng { $$ = append_lng(L(), $1); }
+ | fwf_widthlist ',' poslng
+ { $$ = append_lng($1, $3); }
+ ;
+
+
opt_header_list:
/* empty */ { $$ = NULL; }
| '(' header_list ')' { $$ = $2; }
diff --git a/sql/server/sql_scan.c b/sql/server/sql_scan.c
--- a/sql/server/sql_scan.c
+++ b/sql/server/sql_scan.c
@@ -176,6 +176,8 @@ scanner_init_keywords(void)
keywords_insert("FOR", FOR);
keywords_insert("FOREIGN", FOREIGN);
keywords_insert("FROM", FROM);
+ keywords_insert("FWF", FWF);
+
keywords_insert("REFERENCES", REFERENCES);
keywords_insert("MATCH", MATCH);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
