[Metakit] find select and search, help once again please

Riccardo Cohen Mon, 29 Sep 2003 09:19:13 -0700

I'am puzzled.... I need to do a search, in a table of 15000 records. Below is a sample code and result output

Summary:

Search()
========
=> many errors like :
error search for key '836 my key 836' found idx 15000

this key is in the table, I see it with kitviewer, 3 records have this key.

=> when found, the value is sometimes not the good one : idx=498, key='195 my key 195', foundidx=5850, val='value for key 1950 [dum=5850]', avg=0.188377

it should be the value for key 195, not for key 1950 !

=> If I increase the TOTAL to 10000 instead of 5000, then every search is found with no error !

=> the search is very quick, but does not provide the result !

what does it do exactly ???

Find() ====== result ok, but quite slow. If I hash my table, the result is very quick, but I cant have multiple key ! (which is a problem for me)

Select() ======== result ok, but even slower. and If I hash the table, then the result is better but still very slow. (3 times quicker)

CONCLUSION: =========== - I can't do a quick search in a multiple key table - I can't do a quick search for a range (functin SelectRange uses the same process as Select)

I dont need the performance of a Cray II running an Oracle Server, but 62ms per selection is too much for me (it takes 6 seconds for 100 searches !). Could anybody help me please ?


Thanks
--
Riccardo Cohen

Articque
Les Roches
37230 Fondettes
France
web = http://www.articque.com
tel: +33 02 47 49 90 49
fax: +33 02 47 49 91 49

==========================================================

#ifdef ART_DEBUG
#define TOTAL 1000
#define TOTALMODULO (TOTAL/10)
#define TOTALSEARCH 100
#define TOTALSEARCHMODULO (TOTALSEARCH/10)
#else
#define TOTAL 5000
#define TOTALMODULO (TOTAL/3)
#define TOTALSEARCH 500
#define TOTALSEARCHMODULO (TOTALSEARCH/3)
#endif

// search perfs void execute3() { remove("testperf.db"); c4_StringProp key("key"); c4_StringProp val("val"); c4_Row addrow,findrow; c4_Storage database("testperf.db", true); c4_View strings=database.GetAs("strings[key:S,val:S]"),selection; // TEST HASHING //c4_View viewsec=database.GetAs("sec[_H:I,_R:I]"); //strings=strings.Hash(viewsec,1); // char strkey[500],strval[500]; unsigned long idx,idx2,totaltime,counttime,time,randomnb; long foundidx; struct _timeb tm1,tm2; printf("add\n"); totaltime=0; counttime=0; for (idx=0;idx<TOTAL;idx++) { // sprintf(strkey,"%d my key %d",idx,idx); sprintf(strval,"value for key %d [dum=%d]",idx,3*idx); key(addrow)=strkey; val(addrow)=strval; _ftime(&tm1); strings.Add(addrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; // sprintf(strkey,"%d my key %d",idx,idx); sprintf(strval,"value for key %d [dum=%d]",idx,3*idx+1); key(addrow)=strkey; val(addrow)=strval; _ftime(&tm1); strings.Add(addrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; // sprintf(strkey,"%d my key %d",idx,idx); sprintf(strval,"value for key %d [dum=%d]",idx,3*idx+2); key(addrow)=strkey; val(addrow)=strval; _ftime(&tm1); strings.Add(addrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; if (idx%TOTALMODULO==0 && idx>0) { printf("idx=%d\n",idx); } } printf("avg=%f ms/add\n",(double)totaltime/(double)counttime); _ftime(&tm1); database.Commit(); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); printf("commit=%d ms/add\n",time); printf("search\n"); totaltime=0; counttime=0; for (idx=0;idx<TOTALSEARCH;idx++) { randomnb=rand()%TOTAL; sprintf(strkey,"%d my key %d",randomnb,randomnb); key(findrow)=strkey; _ftime(&tm1); foundidx=strings.Search(findrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; if (foundidx<0||foundidx>=strings.GetSize()) printf("error search for key '%s' found idx %d\n",strkey,foundidx); if (idx%TOTALSEARCHMODULO==0 && idx>0) { printf("idx=%d, key='%s', foundidx=%d, val='%s', avg=%f\n",idx,strkey,foundidx, (const char*)val(strings[foundidx]),(double)totaltime/(double)counttime); } } printf("avg=%f ms/search\n",(double)totaltime/(double)counttime); printf("find\n"); totaltime=0; counttime=0; for (idx=0;idx<TOTALSEARCH;idx++) { randomnb=rand()%TOTAL; sprintf(strkey,"%d my key %d",randomnb,randomnb); key(findrow)=strkey; _ftime(&tm1); foundidx=strings.Find(findrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; if (foundidx<0||foundidx>=strings.GetSize()) printf("error find for key '%s' found idx %d\n",strkey,foundidx); if (idx%TOTALSEARCHMODULO==0 && idx>0) { printf("idx=%d, key='%s', foundidx=%d, val='%s', avg=%f\n",idx,strkey,foundidx, (const char*)val(strings[foundidx]),(double)totaltime/(double)counttime); } while(foundidx>=0) { _ftime(&tm1); foundidx=strings.Find(findrow,foundidx+1); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; if (foundidx>=0 && idx%TOTALSEARCHMODULO==0 && idx>0) printf("idx=%d, key='%s', foundidx=%d, val='%s', avg=%f\n",idx,strkey,foundidx, (const char*)val(strings[foundidx]),(double)totaltime/(double)counttime); } } printf("avg=%f ms/find\n",(double)totaltime/(double)counttime); printf("select\n"); totaltime=0; counttime=0; for (idx=0;idx<TOTALSEARCH;idx++) { randomnb=rand()%TOTAL; sprintf(strkey,"%d my key %d",randomnb,randomnb); key(findrow)=strkey; _ftime(&tm1); selection=strings.Select(findrow); _ftime(&tm2); time=(tm2.time*1000+tm2.millitm)-(tm1.time*1000+tm1.millitm); totaltime+=time; counttime++; if (selection.GetSize()==0) printf("error selection for key '%s' found 0 items\n",strkey,foundidx); if (idx%TOTALSEARCHMODULO==0 && idx>0) { for (idx2=0;(long)idx2<selection.GetSize();idx2++) printf("idx=%d, key='%s', val='%s', avg=%f\n",idx,strkey, (const char*)val(selection[idx2]),(double)totaltime/(double)counttime); } } printf("avg=%f ms/search\n",(double)totaltime/(double)counttime); }


/*
Execute3 WinXP/2P500

#define TOTAL 5000
#define TOTALMODULO (TOTAL/3)
#define TOTALSEARCH 500
#define TOTALSEARCHMODULO (TOTALSEARCH/3)

test nb ?3 add idx=1666 idx=3332 idx=4998 avg=0.011400 ms/add commit=171 ms/add search error search for key '724 my key 724' found idx 15000 error search for key '705 my key 705' found idx 15000 error search for key '667 my key 667' found idx 15000 error search for key '547 my key 547' found idx 15000 error search for key '778 my key 778' found idx 15000 error search for key '890 my key 890' found idx 15000 error search for key '6 my key 6' found idx 15000 error search for key '537 my key 537' found idx 15000 error search for key '829 my key 829' found idx 15000 error search for key '573 my key 573' found idx 15000 error search for key '97 my key 97' found idx 15000 error search for key '574 my key 574' found idx 15000 error search for key '900 my key 900' found idx 15000 error search for key '537 my key 537' found idx 15000 idx=166, key='2595 my key 2595', foundidx=7785, val='value for key 2595 [dum=7785]', avg=0.281437 error search for key '836 my key 836' found idx 15000 error search for key '53 my key 53' found idx 15000 error search for key '600 my key 600' found idx 15000 error search for key '844 my key 844' found idx 15000 error search for key '523 my key 523' found idx 15000 error search for key '580 my key 580' found idx 15000 error search for key '798 my key 798' found idx 15000 error search for key '55 my key 55' found idx 15000 error search for key '699 my key 699' found idx 15000 error search for key '75 my key 75' found idx 15000 error search for key '712 my key 712' found idx 15000 error search for key '585 my key 585' found idx 15000 error search for key '932 my key 932' found idx 15000 error search for key '721 my key 721' found idx 15000 error search for key '555 my key 555' found idx 15000 error search for key '996 my key 996' found idx 15000 idx=332, key='2455 my key 2455', foundidx=7365, val='value for key 2455 [dum=7365]', avg=0.189189 error search for key '734 my key 734' found idx 15000 error search for key '671 my key 671' found idx 15000 error search for key '786 my key 786' found idx 15000 error search for key '53 my key 53' found idx 15000 error search for key '912 my key 912' found idx 15000 error search for key '808 my key 808' found idx 15000 error search for key '945 my key 945' found idx 15000 error search for key '535 my key 535' found idx 15000 error search for key '649 my key 649' found idx 15000 error search for key '824 my key 824' found idx 15000 error search for key '874 my key 874' found idx 15000 error search for key '527 my key 527' found idx 15000 error search for key '974 my key 974' found idx 15000 error search for key '833 my key 833' found idx 15000 error search for key '760 my key 760' found idx 15000 error search for key '627 my key 627' found idx 15000 error search for key '629 my key 629' found idx 15000 idx=498, key='195 my key 195', foundidx=5850, val='value for key 1950 [dum=5850]', avg=0.188377 avg=0.188000 ms/search find idx=166, key='2044 my key 2044', foundidx=6132, val='value for key 2044 [dum=6132]', avg=9.962406 idx=166, key='2044 my key 2044', foundidx=6133, val='value for key 2044 [dum=6133]', avg=9.947447 idx=166, key='2044 my key 2044', foundidx=6134, val='value for key 2044 [dum=6134]', avg=9.932534 idx=332, key='1948 my key 1948', foundidx=5844, val='value for key 1948 [dum=5844]', avg=9.945824 idx=332, key='1948 my key 1948', foundidx=5845, val='value for key 1948 [dum=5845]', avg=9.938346 idx=332, key='1948 my key 1948', foundidx=5846, val='value for key 1948 [dum=5846]', avg=9.930879 idx=498, key='116 my key 116', foundidx=348, val='value for key 116 [dum=348]', avg=9.940793 idx=498, key='116 my key 116', foundidx=349, val='value for key 116 [dum=349]', avg=9.935807 idx=498, key='116 my key 116', foundidx=350, val='value for key 116 [dum=350]', avg=9.930827 avg=9.953000 ms/find select idx=166, key='3664 my key 3664', val='value for key 3664 [dum=10992]', avg=62.407186 idx=166, key='3664 my key 3664', val='value for key 3664 [dum=10993]', avg=62.407186 idx=166, key='3664 my key 3664', val='value for key 3664 [dum=10994]', avg=62.407186 idx=332, key='4901 my key 4901', val='value for key 4901 [dum=14703]', avg=62.078078 idx=332, key='4901 my key 4901', val='value for key 4901 [dum=14704]', avg=62.078078 idx=332, key='4901 my key 4901', val='value for key 4901 [dum=14705]', avg=62.078078 idx=498, key='1163 my key 1163', val='value for key 1163 [dum=3489]', avg=62.655311 idx=498, key='1163 my key 1163', val='value for key 1163 [dum=3490]', avg=62.655311 idx=498, key='1163 my key 1163', val='value for key 1163 [dum=3491]', avg=62.655311 avg=62.686000 ms/search End

*/


_______________________________________________
metakit mailing list  -  [EMAIL PROTECTED]
http://www.equi4.com/mailman/listinfo/metakit

[Metakit] find select and search, help once again please

Reply via email to