Hi Drake, Hi Richard, Hi Paul, Thanks for all your comments, I really appreciate the help.
Please see FULL code below. You can literally copy and paste this and compile it with something like: gcc -m64 -Wall -g -I./ -c ./main.c -o ./bigbyte.o gcc -o ./bigbyte ./bigbyte.o -L./ -Wl,-rpath,./ -lsqlite3 -lpthread -ldl Then if you test it with a file called "in.txt" that has a NUL in it you will see that it cuts it off in the SQLite db "test_queue".... Can you perhaps see something I am doing wrong below? C is not my day job ......as you can see ;-) #include <stdio.h> #include <stdlib.h> #include <string.h> #include "sqlite3.h" #include <time.h> typedef struct messageStruct { char tid[37]; void *raw_stream_in; int num_bytes_in; void *raw_stream_out; int num_bytes_out; char timestamp[20]; }message; sqlite3_stmt *stmt; static sqlite3 *handle; int rc; void* p; char *queue; int queue_createQueue(char *queueName); int queue_openQueue(char *queueName); int queue_closeQueue(); int queue_send(message msg); int main(){ char *buffer = NULL; time_t t; struct tm *tmp; FILE *fp; if((fp = fopen("./in.txt", "rb"))==NULL){ printf("Cannot open file2\n"); exit(1); } fseek(fp, 0, SEEK_END); long int fsize = ftell(fp); printf("File size: %i\n", fsize); rewind(fp); buffer = malloc(fsize * sizeof(char)); fread(buffer, 1, fsize, fp); fclose(fp); //Create Queue rc = queue_createQueue("test_queue"); if (rc ==0){ printf("Queue created...\n"); } message msg_in = {}; strcpy(msg_in.tid, "123"); msg_in.tid[36] = '\0'; msg_in.raw_stream_in = buffer; msg_in.num_bytes_in = fsize; msg_in.raw_stream_out = NULL; msg_in.num_bytes_out = 0; t = time(NULL); tmp = localtime(&t); if (tmp == NULL) { exit(1); } strftime(msg_in.timestamp, sizeof(msg_in.timestamp), "%Y-%m-%d %H:%M:%S", tmp); rc = queue_send(msg_in); if(rc == 0){ printf("Message sent to queue successfully\n"); } rc = queue_closeQueue(); if (rc ==0){ printf("Queue closed...\n"); } return 0; } //Queue handler while waiting to get queue lock int queueBusyHandler(void* db, int count){ return 1; } //Create Queue int queue_createQueue(char *queueName){ char *queueTable = "CREATE TABLE [queue] ( " "[idx] INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " "[tid] CHAR(24) NOT NULL, " "[raw_stream_in] BLOB, " "[num_bytes_in] INTEGER " "[raw_stream_out] BLOB, " "[num_bytes_out] INTEGER " "[timestamp] CHAR(20));"; char *index = "CREATE UNIQUE INDEX tid on queue([tid]);"; char *journalMode = "PRAGMA journal_mode=wal;"; int successFlag = 1; rc = sqlite3_open_v2(queueName,&handle, SQLITE_OPEN_READWRITE | SQLITE_OPEN_SHAREDCACHE | SQLITE_OPEN_CREATE, NULL); if (rc == 0){ p = sqlite3_malloc(256); sqlite3_busy_handler(handle, &queueBusyHandler, p); sqlite3_free(p); sqlite3_exec(handle,"PRAGMA default_cache_size = 50;",0,0,0); rc = sqlite3_exec(handle,journalMode,0,0,0); if(rc == 0){ rc = sqlite3_exec(handle,queueTable,0,0,0); if(rc == 0){ rc = sqlite3_exec(handle,index,0,0,0); if (rc == 0){ successFlag = 0; } } } } return successFlag; } //Close Queue int queue_closeQueue(){ rc = sqlite3_close(handle); return rc; } //Send int queue_send(message msg){ int idx; char *sendText = "INSERT INTO queue (tid, raw_stream_in, num_bytes_in) " "VALUES(:tid, :raw_stream_in, :num_bytes_in)"; sqlite3_exec(handle,"BEGIN IMMEDIATE TRANSACTION;",0,0,0); sqlite3_prepare_v2(handle,sendText,-1,&stmt,0 ); idx = sqlite3_bind_parameter_index( stmt, ":tid" ); sqlite3_bind_text( stmt, idx, msg.tid, -1, SQLITE_STATIC ); idx = sqlite3_bind_parameter_index( stmt, ":raw_stream_in" ); sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in, SQLITE_TRANSIENT ); idx = sqlite3_bind_parameter_index( stmt, ":num_bytes_in" ); sqlite3_bind_int( stmt, idx, msg.num_bytes_in); rc = sqlite3_step(stmt); rc = sqlite3_exec(handle,"COMMIT TRANSACTION;",0,0,0); rc = sqlite3_finalize(stmt); return 0; } On 03/04/2011 13:11, Drake Wilson wrote: > Quoth Lynton Grice<lynton.gr...@logosworld.com>, on 2011-04-03 12:37:06 +0200: >> There are NULL characters > You mean NUL characters. > >> in the first couple header fields [...] >> >> I am no C expert but I have the following table defined: >> >> char *queueTable = "CREATE TABLE [test] ( " >> "[idx] INTEGER NOT NULL PRIMARY KEY >> AUTOINCREMENT, " >> "[raw_stream_in] BLOB, " >> "[num_bytes_in] INTEGER );"; > I wouldn't recommend the use of [] for quoting identifiers (I'd prefer > double-quotes for more ANSI-esque SQL), but that looks like it should > work fine. > >> sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in, >> SQLITE_STATIC ); > Be careful with SQLITE_STATIC. AIUI, the lifetime of the buffer must > then be a superset of the lifetime of the statement. > >> For testing purposes I simply read the 15MB file from file into a char *. >> >> char *buffer = NULL; >> FILE *fp; >> if((fp = fopen("./in.txt", "r"))==NULL){ >> exit(1); >> } > Not "rb" for binary mode? > >> fseek(fp, 0, SEEK_END); >> long int fsize = ftell(fp); >> printf("File size: %i\n", fsize); >> rewind(fp); >> buffer = malloc(fsize * sizeof(char)); >> fread(buffer, 1, fsize, fp); >> buffer[fsize] = '\0'; > You're overwriting beyond the end of the array. You don't need an > extra NUL here. > >> fclose(fp); >> >> I then point the MSG "raw_stream_in" to the buffer: >> >> msg_in.raw_stream_in = buffer; > And you set msg.num_bytes_in where? > >> I then use the following statement as mentioned before to insert the >> stream into the BLOB field: >> >> sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in, >> SQLITE_STATIC ); > That looks fine by itself, subject to the caveats of SQLITE_STATIC > above and whether stmt and idx are valid. > >> _So essentially all characters are copied until the first NULL >> char._ > No. I strongly suspect that's a red herring. > > In summary: > > - Make sure msg.num_bytes_in is actually set to what you want. > - Make sure you're handling the lifetime of the buffer correctly; > for testing purposes I'd use SQLITE_TRANSIENT rather than > SQLITE_STATIC, since that evades that issue at the cost of a > memcpy. > - Don't write past the end of the array. > > ---> Drake Wilson > _______________________________________________ > sqlite-users mailing list > sqlite-users@sqlite.org > http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users _______________________________________________ sqlite-users mailing list sqlite-users@sqlite.org http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users