So I was able to identify the record doing the problem. I have tried to
find what is so special on that record and it is size 20 characters.
When I have this 3 examples with last name lenght 19, 20 and 21
characters only the size 20 crash.
.load ./libstringmetrics.so
select a.firstname, b.firstname, a.lastname, b.lastname,
stringmetrics("qgrams_distance","similarity",a.firstname,
b.firstname,"") first_dist,
stringmetrics("qgrams_distance","similarity",a.lastname, b.lastname,"")
last_dist
from
(select "Milan" as firstname, "Roubal" as lastname ) a,
(select "Milan" as firstname, "RoubalRoubalRoubalR" as lastname ) b
;
.load ./libstringmetrics.so
select a.firstname, b.firstname, a.lastname, b.lastname,
stringmetrics("qgrams_distance","similarity",a.firstname,
b.firstname,"") first_dist,
stringmetrics("qgrams_distance","similarity",a.lastname, b.lastname,"")
last_dist
from
(select "Milan" as firstname, "Roubal" as lastname ) a,
(select "Milan" as firstname, "RoubalRoubalRoubalRo" as lastname ) b
;
.load ./libstringmetrics.so
select a.firstname, b.firstname, a.lastname, b.lastname,
stringmetrics("qgrams_distance","similarity",a.firstname,
b.firstname,"") first_dist,
stringmetrics("qgrams_distance","similarity",a.lastname, b.lastname,"")
last_dist
from
(select "Milan" as firstname, "Roubal" as lastname ) a,
(select "Milan" as firstname, "RoubalRoubalRoubalRou" as lastname ) b
;
I got also once from 30 crashes this trace, maybe it will help to find
the problem.
*** Error in `/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3':
free(): invalid next size (fast): 0x0000000001b53cf0 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x73d8f)[0x7ffa14020d8f]
/lib64/libc.so.6(+0x795ee)[0x7ffa140265ee]
/lib64/libc.so.6(+0x7a2c7)[0x7ffa140272c7]
./libstringmetrics.so(qgram_tokenize_to_utarray+0x2c0)[0x7ffa13b6b5e0]
./libstringmetrics.so(qgrams_distance_similarity_custom+0x27)[0x7ffa13b69257]
./libstringmetrics.so(qgrams_distance_similarity+0x37)[0x7ffa13b693b7]
./libstringmetrics.so(stringmetricsFunc+0xcfb)[0x7ffa13b62b9b]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0(+0x6dc74)[0x7ffa14a11c74]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0(sqlite3_step+0x28f)[0x7ffa14a163ff]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3[0x406ee1]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3[0x407fe3]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3[0x40337b]
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7ffa13fcebe5]
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3[0x403b69]
======= Memory map: ========
00400000-00410000 r-xp 00000000 00:27 50993
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3
0060f000-00610000 r--p 0000f000 00:27 50993
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3
00610000-00612000 rw-p 00010000 00:27 50993
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3
01b3f000-01df2000 rw-p 00000000 00:00 0
[heap]
7ffa13644000-7ffa1365a000 r-xp 00000000 00:14 1355
/lib64/libgcc_s.so.1
7ffa1365a000-7ffa13859000 ---p 00016000 00:14 1355
/lib64/libgcc_s.so.1
7ffa13859000-7ffa1385a000 r--p 00015000 00:14 1355
/lib64/libgcc_s.so.1
7ffa1385a000-7ffa1385b000 rw-p 00016000 00:14 1355
/lib64/libgcc_s.so.1
7ffa1385b000-7ffa1395d000 r-xp 00000000 00:14 572971
/lib64/libm-2.18.so
7ffa1395d000-7ffa13b5c000 ---p 00102000 00:14 572971
/lib64/libm-2.18.so
7ffa13b5c000-7ffa13b5d000 r--p 00101000 00:14 572971
/lib64/libm-2.18.so
7ffa13b5d000-7ffa13b5e000 rw-p 00102000 00:14 572971
/lib64/libm-2.18.so
7ffa13b5e000-7ffa13b6f000 r-xp 00000000 00:27 52106
/home/milan/sqlite/libstringmetrics.so
7ffa13b6f000-7ffa13d6f000 ---p 00011000 00:27 52106
/home/milan/sqlite/libstringmetrics.so
7ffa13d6f000-7ffa13d70000 r--p 00011000 00:27 52106
/home/milan/sqlite/libstringmetrics.so
7ffa13d70000-7ffa13d71000 rw-p 00012000 00:27 52106
/home/milan/sqlite/libstringmetrics.so
7ffa13d71000-7ffa13d79000 rw-p 00000000 00:00 0
7ffa13d79000-7ffa13da4000 r-xp 00000000 00:14 1348
/lib64/libtinfo.so.5.9
7ffa13da4000-7ffa13fa3000 ---p 0002b000 00:14 1348
/lib64/libtinfo.so.5.9
7ffa13fa3000-7ffa13fa7000 r--p 0002a000 00:14 1348
/lib64/libtinfo.so.5.9
7ffa13fa7000-7ffa13fac000 rw-p 0002e000 00:14 1348
/lib64/libtinfo.so.5.9
7ffa13fac000-7ffa13fad000 rw-p 00000000 00:00 0
7ffa13fad000-7ffa14152000 r-xp 00000000 00:14 572963
/lib64/libc-2.18.so
7ffa14152000-7ffa14351000 ---p 001a5000 00:14 572963
/lib64/libc-2.18.so
7ffa14351000-7ffa14355000 r--p 001a4000 00:14 572963
/lib64/libc-2.18.so
7ffa14355000-7ffa14357000 rw-p 001a8000 00:14 572963
/lib64/libc-2.18.so
7ffa14357000-7ffa1435b000 rw-p 00000000 00:00 0
7ffa1435b000-7ffa14373000 r-xp 00000000 00:14 572991
/lib64/libpthread-2.18.so
7ffa14373000-7ffa14573000 ---p 00018000 00:14 572991
/lib64/libpthread-2.18.so
7ffa14573000-7ffa14574000 r--p 00018000 00:14 572991
/lib64/libpthread-2.18.so
7ffa14574000-7ffa14575000 rw-p 00019000 00:14 572991
/lib64/libpthread-2.18.so
7ffa14575000-7ffa14579000 rw-p 00000000 00:00 0
7ffa14579000-7ffa1457c000 r-xp 00000000 00:14 572969
/lib64/libdl-2.18.so
7ffa1457c000-7ffa1477b000 ---p 00003000 00:14 572969
/lib64/libdl-2.18.so
7ffa1477b000-7ffa1477c000 r--p 00002000 00:14 572969
/lib64/libdl-2.18.so
7ffa1477c000-7ffa1477d000 rw-p 00003000 00:14 572969
/lib64/libdl-2.18.so
7ffa1477d000-7ffa147a3000 r-xp 00000000 00:14 1463
/lib64/libncurses.so.5.9
7ffa147a3000-7ffa149a2000 ---p 00026000 00:14 1463
/lib64/libncurses.so.5.9
7ffa149a2000-7ffa149a3000 r--p 00025000 00:14 1463
/lib64/libncurses.so.5.9
7ffa149a3000-7ffa149a4000 rw-p 00026000 00:14 1463
/lib64/libncurses.so.5.9
7ffa149a4000-7ffa14a63000 r-xp 00000000 00:27 50970
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0.8.6
7ffa14a63000-7ffa14c62000 ---p 000bf000 00:27 50970
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0.8.6
7ffa14c62000-7ffa14c64000 r--p 000be000 00:27 50970
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0.8.6
7ffa14c64000-7ffa14c67000 rw-p 000c0000 00:27 50970
/home/milan/sqlite/sqlite-autoconf-3080803/.libs/libsqlite3.so.0.8.6
7ffa14c67000-7ffa14c87000 r-xp 00000000 00:14 572956
/lib64/ld-2.18.so
7ffa14e28000-7ffa14e5d000 r--s 00000000 00:10 8251
/var/run/nscd/passwd
7ffa14e5d000-7ffa14e62000 rw-p 00000000 00:00 0
7ffa14e82000-7ffa14e86000 rw-p 00000000 00:00 0
7ffa14e86000-7ffa14e87000 r--p 0001f000 00:14 572956
/lib64/ld-2.18.so
7ffa14e87000-7ffa14e88000 rw-p 00020000 00:14 572956
/lib64/ld-2.18.so
7ffa14e88000-7ffa14e89000 rw-p 00000000 00:00 0
7fff40c81000-7fff40ca2000 rw-p 00000000 00:00 0
[stack]
7fff40d02000-7fff40d04000 r-xp 00000000 00:00 0
[vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0
[vsyscall]
Aborted
Milan
On 2015-03-11 23:18, Milan Roubal wrote:
> Your SQL pass, so it is somewhere in the library. The problem is how
> to find in the big data where exactly. I am now going one character by
> each other, because it is probably only one specific record kind doing
> that.
>
> I was able to compile the library under linux and running strace looks
> like this:
>
> lseek(3, 9742336, SEEK_SET) = 9742336
> read(3,
> "\r\0\0\0.\0x\0\3\357\3\333\3\313\3\273\3\250\3\223\3~\3k\3Y\3G\0033\3\31"...,
> 1024) = 1024
> lseek(3, 9743360, SEEK_SET) = 9743360
> read(3,
> "\r\0\0\0)\0~\0\3\354\3\330\3\304\3\260\3\234\3\210\3t\3`\3L\0038\3$\3\20"...,
> 1024) = 1024
> open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = 4
> writev(4, [{"*** Error in `", 14},
> {"/home/milan/sqlite/sqlite-autoco"..., 56}, {"': ", 3}, {"free():
> invalid next size (fast)", 32}, {": 0x", 4}, {"0000000001eadc20", 16},
> {" ***\n", 5}], 7*** Error in
> `/home/milan/sqlite/sqlite-autoconf-3080803/.libs/sqlite3': free():
> invalid next size (fast): 0x0000000001eadc20 ***
> ) = 130
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7fe78ca59000
>
> any hint how to fast find which record is doing that crash or even fix
> for such crash is welcome :)
> Milan
>
>
> On 2015-03-11 19:30, Simon Slavin wrote:
>>> On 11 Mar 2015, at 4:56pm, Milan Roubal <roubal at keyserver.cz> wrote:
>>>
>>> I am also facing problem with sqlite closed by windows because of
>>> crash. On same data this query crash:
>>>
>>> select load_extension("libstringmetrics.dll");
>>> select a.vorname, b.vorname, a.nachname, b.nachname,
>>> stringmetrics("qgrams_distance","similarity",a.nachname, b.nachname,
>>> "") nach_dist
>>> from r2 a, allUsers b
>>> where
>>> a.vorname like 'r%' and b.vorname like 'r%' and
>>> round(nach_dist) between 50 and 100
>>> ;
>>>
>>> The first query is able to pass on some small sample, but not on all
>>> my data. I am trying to find what concrete string comparison is
>>> crashing it, but it is slow with crashes. Is there any way how to
>>> start logging the calls of the library to find the exact data that
>>> are causing the crash?
>>
>> Try this command
>>
>> select a.vorname, b.vorname, a.nachname, b.nachname,
>> 55 nach_dist
>> from r2 a, allUsers b
>> where
>> a.vorname like 'r%' and b.vorname like 'r%' and
>> round(nach_dist) between 50 and 100
>>
>> If it does not crash then the problem is in your stringmetrics
>> library.
>>
>> Simon.
>> _______________________________________________
>> sqlite-users mailing list
>> sqlite-users at mailinglists.sqlite.org
>> http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
>
> _______________________________________________
> sqlite-users mailing list
> sqlite-users at mailinglists.sqlite.org
> http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users
