Package: release.debian.org Severity: normal User: release.debian....@packages.debian.org Usertags: unblock
Please unblock package dovecot-fts-xapian This version (1.4.9a) fixes a number of important bugs in the indexer including: + fix indexing of attachments, closes: #985654 + fix indexing of accented characters + fix memory errors / segfaults when indexing large mailboxes Source debdiff from 1.4.7-1 (currently in testing) to 1.4.9a-1 is attached here. Please let me know when approved so I can upload to unstable. unblock dovecot-fts-xapian/1.4.9a-1 Thanks, --Joe
diffstat for dovecot-fts-xapian-1.4.7 dovecot-fts-xapian-1.4.9a .gitignore | 65 ++++++++++ Makefile.am | 4 PACKAGES/RPM/README.md | 20 +++ PACKAGES/RPM/fts-xapian.spec | 41 ++++++ README.md | 46 +++++-- configure.ac | 2 debian/changelog | 11 + debian/watch | 4 fts-xapian-config.h.in | 2 src/fts-backend-xapian-functions.cpp | 175 +++++++++++++++++++++-------- src/fts-backend-xapian.cpp | 211 ++++++++++++++++++----------------- src/fts-xapian-plugin.c | 2 src/fts-xapian-plugin.h | 9 - 13 files changed, 425 insertions(+), 167 deletions(-) diff -Nru -w dovecot-fts-xapian-1.4.7/.gitignore dovecot-fts-xapian-1.4.9a/.gitignore --- dovecot-fts-xapian-1.4.7/.gitignore 1969-12-31 19:00:00.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/.gitignore 2021-04-24 16:27:55.000000000 -0400 @@ -0,0 +1,65 @@ +# http://www.gnu.org/software/automake + +Makefile.in +/ar-lib +/mdate-sh +/py-compile +/test-driver +/ylwrap +.deps/ +.dirstamp + +# http://www.gnu.org/software/autoconf + +autom4te.cache +/autoscan.log +/autoscan-*.log +/aclocal.m4 +/compile +/config.guess +/config.h.in +/config.log +/config.status +/config.sub +/configure +/configure.scan +/depcomp +/install-sh +/missing +/stamp-h1 +/stamp-h2 +/stamp.h + +# https://www.gnu.org/software/libtool/ + +/ltmain.sh +/libtool + +# http://www.gnu.org/software/texinfo + +/texinfo.tex + +# http://www.gnu.org/software/m4/ + +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 + +# Generated Makefile +# (meta build system like autotools, +# can automatically generate from config.status script +# (which is called by configure script)) +Makefile + +/dummy-config.h +/dummy-config.h.in +/fts-xapian-config.h +/run-test.sh + +src/*.o +src/*.lo +src/*.la + +src/.libs/** diff -Nru -w dovecot-fts-xapian-1.4.7/Makefile.am dovecot-fts-xapian-1.4.9a/Makefile.am --- dovecot-fts-xapian-1.4.7/Makefile.am 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/Makefile.am 2021-04-24 16:27:55.000000000 -0400 @@ -2,5 +2,5 @@ ACLOCAL_AMFLAGS = -I m4 -PACKAGE_VERSION = "1.4.7" -VERSION = "1.4.7" +PACKAGE_VERSION = "1.4.9a" +VERSION = "1.4.9a" diff -Nru -w dovecot-fts-xapian-1.4.7/PACKAGES/RPM/README.md dovecot-fts-xapian-1.4.9a/PACKAGES/RPM/README.md --- dovecot-fts-xapian-1.4.7/PACKAGES/RPM/README.md 1969-12-31 19:00:00.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/PACKAGES/RPM/README.md 2021-04-24 16:27:55.000000000 -0400 @@ -0,0 +1,20 @@ +As root: + + Install the development environment and required devel packages: + -- dnf groupinstall "Development Tools" + -- dnf install rpm-build rpm-devel rpmlint make coreutils diffutils patch rpmdevtools + -- dnf install dovecot-devel dovecot libicu-devel icu xapian-core xapian-core-devel + +As a normal user: + + Create the ~/rpmbuild tree as a normal user (never build rpms as root): + -- rpmdev-setuptree + Place the spec file under: + ~/rpmbuild/SPECS/fts-xapian.spec + Place the tar.gz sources under: + ~/rpmbuild/SOURCES/fts-xapian-1.4.9a.tar.gz + Generate the binary rpm with: + -- QA_RPATHS=$(( 0x0001|0x0010 )) rpmbuild -bb ~/rpmbuild/SPECS/fts-xapian.spec + +Your RPM packages will be under ~/rpmbuild/RPMS/x86_64/ + diff -Nru -w dovecot-fts-xapian-1.4.7/PACKAGES/RPM/fts-xapian.spec dovecot-fts-xapian-1.4.9a/PACKAGES/RPM/fts-xapian.spec --- dovecot-fts-xapian-1.4.7/PACKAGES/RPM/fts-xapian.spec 1969-12-31 19:00:00.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/PACKAGES/RPM/fts-xapian.spec 2021-04-24 16:27:55.000000000 -0400 @@ -0,0 +1,41 @@ +Name: fts-xapian +Version: 1.4.9a +Release: 1%{?dist} +Summary: Dovecot FTS plugin based on Xapian + +License: LGPL-2.1 +URL: https://github.com/grosjo/fts-xapian +Source0: fts-xapian-1.4.9a.tar.gz + +BuildRequires: xapian-core-devel, libicu-devel, dovecot-devel +Requires: xapian-core, xapian-core-libs, dovecot + +%description +This project intends to provide a straightforward, simple and maintenance free, way to configure FTS plugin for Dovecot, leveraging the efforts by the Xapian.org team. + +This effort came after Dovecot team decided to deprecate "fts_squat" included in the dovecot core, and due to the complexity of the Solr plugin capabilitles, un-needed for most users. + + +%prep +%autosetup +autoreconf -vi +./configure --with-dovecot=/usr/lib64/dovecot + + +%build +make %{?_smp_mflags} + + +%install +%make_install + + +%files +/usr/lib64/dovecot/lib21_fts_xapian_plugin.la +/usr/lib64/dovecot/lib21_fts_xapian_plugin.so +/usr/lib64/dovecot/lib21_fts_xapian_plugin.a + + +%changelog +* Tue Apr 6 2021 xapian +- diff -Nru -w dovecot-fts-xapian-1.4.7/README.md dovecot-fts-xapian-1.4.9a/README.md --- dovecot-fts-xapian-1.4.7/README.md 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/README.md 2021-04-24 16:27:55.000000000 -0400 @@ -79,12 +79,14 @@ plugin = fts fts_xapian (...) fts = xapian - fts_xapian = partial=3 full=20 attachments=0 verbose=0 + fts_xapian = partial=3 full=20 verbose=0 fts_autoindex = yes fts_enforced = yes fts_autoindex_exclude = \Trash + + fts_decoder = decode2text // To index attachements (...) } @@ -92,27 +94,48 @@ service indexer-worker { vsz_limit = 2G // or above (or 0 if you have rather large memory usable on your server, which is preferred for performance) } + +service decode2text { + executable = script /usr/libexec/dovecot/decode2text.sh + user = dovecot + unix_listener decode2text { + mode = 0666 + } +} (...) ``` -Partial & full parameters : 3 and 20 are the NGram values for header fields, which means the keywords created for fields (To, Cc, ...) are between 3 and 20 chars long. -Full words are also added by default (if not longer than 245 chars, which is the limit of Xapian capability). + +Indexing options +---------------- + +| Option | Description | Possible values | Default value | +|----------------|--------------------------------|--------------------------------------|---------------| +| partial & full | NGram values for header fields | between 3 and 20 characters | 3 & 20 | +| verbose | Logs verbosity | 0 (silent), 1 (verbose) or 2 (debug) | 0 | + +NGrams details +-------------- + +The partial & full parameters are the NGram values for header fields, which means the keywords created for fields (To, +Cc, ...) are between 3 and 20 chars long. Full words are also added by default (if not longer than 245 chars, which is +the limit of Xapian capability). Example: "<john@doe>" will create joh, ohn, hn@, ..., john@d, ohn@do, ..., and finally john@doe as searchable keywords. -Set "verbose=1" to see verbose messages in the log, "verbose=2" for debug -Set "attachments=1" if you want to index attachments (this works only for text attachments) +Index updating +-------------- -Restart Dovecot: +Just restart Dovecot: -``` +```sh sudo servicectl restart dovecot ``` -If this is not a fresh install of dovecot, you need to re-index your mailboxes +If this is not a fresh install of dovecot, you need to re-index your mailboxes: -``` +```sh doveadm index -A -q \* ``` @@ -121,7 +144,8 @@ You shall put in a cron the following command (for daily run for instance) : -``` + +```sh doveadm fts optimize -A ``` @@ -131,4 +155,6 @@ Please submit requests/bugs via the [GitHub issue tracker](https://github.com/grosjo/fts-xapian/issues). +A Matrix Room exists also at : #xapian-dovecot:grosjo.net + Thanks to Aki Tuomi <aki.tu...@open-xchange.com>, Stephan Bosch <step...@rename-it.nl>, Paul Hecker <p...@iwascoding.com> diff -Nru -w dovecot-fts-xapian-1.4.7/configure.ac dovecot-fts-xapian-1.4.9a/configure.ac --- dovecot-fts-xapian-1.4.7/configure.ac 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/configure.ac 2021-04-24 16:27:55.000000000 -0400 @@ -1,4 +1,4 @@ -AC_INIT([Dovecot FTS Xapian], [1.4.7], [j...@grosjo.net], [dovecot-fts-xapian]) +AC_INIT([Dovecot FTS Xapian], [1.4.9a], [j...@grosjo.net], [dovecot-fts-xapian]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) diff -Nru -w dovecot-fts-xapian-1.4.7/debian/changelog dovecot-fts-xapian-1.4.9a/debian/changelog --- dovecot-fts-xapian-1.4.7/debian/changelog 2021-01-31 21:35:02.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/debian/changelog 2021-06-22 22:40:48.000000000 -0400 @@ -1,3 +1,14 @@ +dovecot-fts-xapian (1.4.9a-1) unstable; urgency=medium + + * [2da6c89] d/watch: allow non-numbers in version + * [18c496d] New upstream version 1.4.9a + + fix indexing of attachments, closes: #985654 + + fix indexing of accented characters + + fix memory errors / segfaults when indexing large mailboxes + + handle indexing of virtual folders + + -- Joseph Nahmias <je...@debian.org> Tue, 22 Jun 2021 22:40:48 -0400 + dovecot-fts-xapian (1.4.7-1) unstable; urgency=medium * [e3bdb1b] fix VCS paths diff -Nru -w dovecot-fts-xapian-1.4.7/debian/watch dovecot-fts-xapian-1.4.9a/debian/watch --- dovecot-fts-xapian-1.4.7/debian/watch 2020-10-18 12:23:30.000000000 -0400 +++ dovecot-fts-xapian-1.4.9a/debian/watch 2021-06-22 22:10:41.000000000 -0400 @@ -3,6 +3,6 @@ version=4 -opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)\.tar\.gz%dovecot-fts-xapian-$1.tar.gz%" \ +opts="filenamemangle=s%(?:.*?)?v?(\d.*)\.tar\.gz%dovecot-fts-xapian-$1.tar.gz%" \ https://github.com/grosjo/fts-xapian/tags \ - (?:.*?/)?v?(\d[\d.]*)\.tar\.gz + (?:.*?/)?v?(\d.*)\.tar\.gz diff -Nru -w dovecot-fts-xapian-1.4.7/fts-xapian-config.h.in dovecot-fts-xapian-1.4.9a/fts-xapian-config.h.in --- dovecot-fts-xapian-1.4.7/fts-xapian-config.h.in 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/fts-xapian-config.h.in 2021-04-24 16:27:55.000000000 -0400 @@ -1,2 +1,2 @@ #define FTS_XAPIAN_NAME "Dovecot FTS Xapian" -#define FTS_XAPIAN_VERSION "1.4.7" +#define FTS_XAPIAN_VERSION "1.4.9a" diff -Nru -w dovecot-fts-xapian-1.4.7/src/fts-backend-xapian-functions.cpp dovecot-fts-xapian-1.4.9a/src/fts-backend-xapian-functions.cpp --- dovecot-fts-xapian-1.4.7/src/fts-backend-xapian-functions.cpp 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/src/fts-backend-xapian-functions.cpp 2021-04-24 16:27:55.000000000 -0400 @@ -26,6 +26,8 @@ class XQuerySet { + private: + icu::Transliterator *accentsConverter; public: char * header; char * text; @@ -44,6 +46,7 @@ header=NULL; text=NULL; global_neg=false; + accentsConverter=NULL; } XQuerySet(bool is_and, bool is_neg, long l) @@ -55,6 +58,7 @@ text=NULL; global_and=is_and; global_neg=is_neg; + accentsConverter=NULL; } ~XQuerySet() @@ -68,6 +72,7 @@ } if(qsize>0) i_free(qs); qsize=0; qs=NULL; + if(accentsConverter != NULL) delete(accentsConverter); } void add(const char * h,const char * t) @@ -80,11 +85,8 @@ if(h==NULL) return; if(t==NULL) return; - icu::StringPiece sp_h(h); - icu::UnicodeString h2 = icu::UnicodeString::fromUTF8(sp_h); - - icu::StringPiece sp_t(t); - icu::UnicodeString t2 = icu::UnicodeString::fromUTF8(sp_t); + icu::UnicodeString h2 = icu::UnicodeString::fromUTF8(icu::StringPiece(h)); + icu::UnicodeString t2 = icu::UnicodeString::fromUTF8(icu::StringPiece(t)); add(&h2,&t2,is_neg); } @@ -154,6 +156,19 @@ std::string tmp1; h->toUTF8String(tmp1); char * h2 = i_strdup(tmp1.c_str()); + + if(accentsConverter == NULL) + { + UErrorCode status = U_ZERO_ERROR; + accentsConverter = icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC", UTRANS_FORWARD, status); + if(U_FAILURE(status)) + { + i_error("FTS Xapian: Can not allocate ICU translator (2)"); + accentsConverter = NULL; + } + } + if(accentsConverter != NULL) accentsConverter->transliterate(*t); + std::string tmp2; t->toUTF8String(tmp2); char * t2 = i_strdup(tmp2.c_str()); @@ -178,7 +193,7 @@ } if(i>=HDRS_NB) { - i_error("FTS Xapian: Unknown header (lookup) '%s'",h2); + if(verbose>1) i_error("FTS Xapian: Unknown header (lookup) '%s'",h2); i_free(h2); i_free(t2); return; } @@ -299,8 +314,6 @@ char *s = i_strdup(get_string().c_str()); - if(verbose>0) { i_info("FTS Xapian: Query= %s",s); } - qp->set_database(*db); Xapian::Query * q = new Xapian::Query(qp->parse_query(s,Xapian::QueryParser::FLAG_DEFAULT));// | Xapian::QueryParser::FLAG_PARTIAL)); @@ -317,6 +330,7 @@ long partial,full,hardlimit; const char * prefix; bool onlyone; + icu::Transliterator *accentsConverter; public: char ** data; @@ -334,6 +348,7 @@ hardlimit=XAPIAN_TERM_SIZELIMIT-strlen(prefix); onlyone=false; if(strcmp(prefix,"XMID")==0) onlyone=true; + accentsConverter = NULL; } ~XNGram() @@ -348,14 +363,14 @@ i_free(data); } data=NULL; + if(accentsConverter != NULL) delete(accentsConverter); } void add(const char * s) { if(s==NULL) return; - icu::StringPiece sp(s); - icu::UnicodeString d = icu::UnicodeString::fromUTF8(sp); + icu::UnicodeString d = icu::UnicodeString::fromUTF8(icu::StringPiece(s)); add(&d); } @@ -401,6 +416,18 @@ long l = d->length(); if(l<partial) return; + if(accentsConverter == NULL) + { + UErrorCode status = U_ZERO_ERROR; + accentsConverter = icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC", UTRANS_FORWARD, status); + if(U_FAILURE(status)) + { + i_error("FTS Xapian: Can not allocate ICU translator (1)"); + accentsConverter = NULL; + } + } + if(accentsConverter != NULL) accentsConverter->transliterate(*d); + if(onlyone) { add_stem(d); @@ -470,6 +497,13 @@ } }; +static long fts_backend_xapian_current_time() +{ + struct timeval tp; + gettimeofday(&tp, NULL); + return tp.tv_sec * 1000 + tp.tv_usec / 1000; +} + static long fts_backend_xapian_memory_used() // KB { FILE* file = fopen("/proc/self/status", "r"); @@ -522,7 +556,7 @@ return 0; } -static bool fts_backend_xapian_test_memory() +static bool fts_backend_xapian_test_memory(struct xapian_fts_backend *backend, long add) { rlim_t limit; @@ -531,15 +565,23 @@ long used = fts_backend_xapian_memory_used(); long fri = fts_backend_xapian_memory_free(); // Free RAM + backend->nb_pushes++; + long m2 = 2*used/backend->nb_pushes; + if(backend->max_push < m2) backend->max_push=m2; + m2=backend->max_push; + + add = long(add/1024.0); + if(m<1) { - if(verbose>0) i_info("FTS Xapian: Memory stats : Used = %ld MB, Free = %ld MB",long(used/1024),long(fri/1024)); - return (fri>used/2); + if(verbose>0) i_info("FTS Xapian: Memory stats : Used = %ld MB (%ld pushes), Free = %ld MB, Additional data %ld KB, Estimated required = %ld MB",long(used/1024), backend->nb_pushes, long(fri/1024), add, long(m2/1024)); + return ((fri>XAPIAN_MIN_RAM*1024)&&(fri>m2)); + } + else + { + if(verbose>0) i_info("FTS Xapian: Memory stats : Used = %ld MB (%ld%%) (%ld pushes), Limit = %ld MB, Free = %ld MB, Additional data %ld KB, Estimated required = %ld MB",long(used/1024),long(used*100.0/m),backend->nb_pushes,long(m/1024),long(fri/1024), add, long(m2/1024)); + return ((fri>XAPIAN_MIN_RAM*1024)&&(m>(used+m2))&&(fri>m2)); } - - if(verbose>0) i_info("FTS Xapian: Memory stats : Used = %ld MB (%ld%%), Limit = %ld MB, Free = %ld MB",long(used/1024),long(used*100.0/m),long(m/1024),long(fri/1024)); - - return ((m>used*3.0/2)&&(fri>used/2)); } static bool fts_backend_xapian_open_readonly(struct xapian_fts_backend *backend, Xapian::Database ** dbr) @@ -598,9 +640,7 @@ if(backend->old_guid != NULL) { /* Performance calculator*/ - struct timeval tp; - gettimeofday(&tp, NULL); - long dt = tp.tv_sec * 1000 + tp.tv_usec / 1000 - backend->perf_dt; + long dt = fts_backend_xapian_current_time() - backend->perf_dt; double r=0; if(dt>0) { @@ -618,8 +658,12 @@ static void fts_backend_xapian_release(struct xapian_fts_backend *backend, const char * reason, long commit_time) { + bool err=false; + if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_release (%s)",reason); + if(commit_time<1) commit_time = fts_backend_xapian_current_time(); + if(backend->dbw !=NULL) { try @@ -630,6 +674,7 @@ catch(Xapian::Error e) { i_error("FTS Xapian: %s : %s - %s",reason,e.get_type(),e.get_error_string()); + err=true; } delete(backend->dbw); backend->dbw = NULL; @@ -637,12 +682,27 @@ backend->commit_time = commit_time; } + if(err) + { + if(verbose>0) i_info("FTS Xapian: Re-creating index database due to error"); + try + { + Xapian::WritableDatabase * db = new Xapian::WritableDatabase(backend->db,Xapian::DB_CREATE_OR_OVERWRITE | Xapian::DB_RETRY_LOCK | Xapian::DB_BACKEND_GLASS); + db->close(); + delete(db); + } + catch(Xapian::Error e) + { + i_error("FTS Xapian: Can't re-create Xapian DB (%s) %s : %s - %s",backend->boxname,backend->db,e.get_type(),e.get_error_string()); + } + } + + backend->nb_pushes=0; + backend->max_push=0; + if(verbose>0) { - struct timeval tp; - gettimeofday(&tp, NULL); - long current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; - i_info("FTS Xapian: Committed '%s' in %ld ms",reason,current_time - commit_time); + i_info("FTS Xapian: Committed '%s' in %ld ms",reason,fts_backend_xapian_current_time() - commit_time); } } @@ -689,9 +749,7 @@ { Xapian::WritableDatabase * dbw; - struct timeval tp; - gettimeofday(&tp, NULL); - long dt = tp.tv_sec * 1000 + tp.tv_usec / 1000; + long dt = fts_backend_xapian_current_time(); try { @@ -747,21 +805,18 @@ dbw->commit(); dbw->close(); delete(dbw); - gettimeofday(&tp, NULL); - dt = tp.tv_sec * 1000 + tp.tv_usec / 1000 - dt; + + dt = fts_backend_xapian_current_time() - dt; i_info("FTS Xapian: Expunging '%s' done in %.2f secs",fpath,dt/1000.0); } static int fts_backend_xapian_unset_box(struct xapian_fts_backend *backend) { - if(verbose>1) i_info("FTS Xapian: Unset box '%s' (%s)",backend->boxname,backend->guid); + if(verbose>0) i_info("FTS Xapian: Unset box '%s' (%s)",backend->boxname,backend->guid); - struct timeval tp; - gettimeofday(&tp, NULL); - long commit_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; + long commit_time = fts_backend_xapian_current_time(); fts_backend_xapian_oldbox(backend); - fts_backend_xapian_release(backend,"unset_box",commit_time); if(backend->db != NULL) @@ -779,19 +834,47 @@ return 0; } +static int fts_backend_xapian_set_path(struct xapian_fts_backend *backend) +{ + struct mail_namespace * ns = backend->backend.ns; + if(ns->alias_for != NULL) + { + if(verbose>0) i_info("FTS Xapian: Switching namespace"); + ns = ns->alias_for; + } + + const char * path = mailbox_list_get_root_forced(ns->list, MAILBOX_LIST_PATH_TYPE_INDEX); + + if(backend->path != NULL) i_free(backend->path); + backend->path = i_strconcat(path, "/" XAPIAN_FILE_PREFIX, NULL); + + if(verbose>0) i_info("FTS Xapian: Index path = %s",backend->path); + + struct stat sb; + if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) + { + if (mailbox_list_mkdir_root(backend->backend.ns->list, backend->path, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) + { + i_error("FTS Xapian: can not create '%s'",backend->path); + return -1; + } + } + return 0; +} + static int fts_backend_xapian_set_box(struct xapian_fts_backend *backend, struct mailbox *box) { if (box == NULL) { if(backend->guid != NULL) fts_backend_xapian_unset_box(backend); - if(verbose>0) i_info("FTS Xapian: Box is empty"); + if(verbose>1) i_info("FTS Xapian: Box is empty"); return 0; } const char * mb; fts_mailbox_get_guid(box, &mb ); - if(verbose>1) i_info("FTX Xapian: Set box '%s' (%s)",box->name,mb); + if(verbose>0) i_info("FTS Xapian: Set box '%s' (%s)",box->name,mb); if((mb == NULL) || (strlen(mb)<3)) { @@ -807,11 +890,12 @@ if(backend->guid != NULL) fts_backend_xapian_unset_box(backend); + if(fts_backend_xapian_set_path(backend)<0) return -1; + struct timeval tp; long current_time; - gettimeofday(&tp, NULL); - current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; + current_time = fts_backend_xapian_current_time(); backend->commit_updates = 0; backend->commit_time = current_time; @@ -837,7 +921,6 @@ } i_free(t); - /* Performance calculator*/ backend->perf_dt = current_time; backend->perf_uid=0; @@ -917,15 +1000,15 @@ { bool ok=true; - if(verbose>1) i_info("FTS Xapian: fts_backend_xapian_index_hdr"); + if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_index_hdr"); Xapian::WritableDatabase * dbx = backend->dbw; long p = backend->partial; long f = backend->full; - if(data->length()<p) { return true; } + if(data->length()<p) return true; - if(strlen(field)<1) { return true; } + if(strlen(field)<1) return true; long i=0; while((i<HDRS_NB) && (strcmp(field,hdrs_emails[i])!=0)) @@ -1006,7 +1089,7 @@ } catch (std::bad_alloc& ba) { - i_error("FTS Xapian: Memory error '%s'",ba.what()); + i_info("FTS Xapian: Memory too low (hdr) '%s'",ba.what()); ok = false; } } @@ -1020,13 +1103,13 @@ { bool ok = true; - if(verbose>1) i_info("FTS Xapian: fts_backend_xapian_index_text"); + if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_index_text"); Xapian::WritableDatabase * dbx = backend->dbw; long p = backend->partial; long f = backend->full; - if(data->length()<p) { return true; } + if(data->length()<p) return true; XQuerySet * xq = new XQuerySet(); @@ -1134,7 +1217,7 @@ } catch (std::bad_alloc& ba) { - i_error("FTS Xapian: Memory error '%s'",ba.what()); + i_info("FTS Xapian: Memory too low (text) '%s'",ba.what()); ok = false; } } diff -Nru -w dovecot-fts-xapian-1.4.7/src/fts-backend-xapian.cpp dovecot-fts-xapian-1.4.9a/src/fts-backend-xapian.cpp --- dovecot-fts-xapian-1.4.7/src/fts-backend-xapian.cpp 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/src/fts-backend-xapian.cpp 2021-04-24 16:27:55.000000000 -0400 @@ -6,7 +6,9 @@ #include "fts-xapian-plugin.h" } #include <dirent.h> +#include <unicode/utypes.h> #include <unicode/unistr.h> +#include <unicode/translit.h> #include <sys/time.h> #define XAPIAN_FILE_PREFIX "xapian-indexes" @@ -15,6 +17,7 @@ #define XAPIAN_COMMIT_TIMEOUT 300L #define XAPIAN_WILDCARD "wldcrd" #define XAPIAN_EXPUNGE_HEADER 9 +#define XAPIAN_MIN_RAM 200L #define HDRS_NB 11 static const char * hdrs_emails[HDRS_NB] = { "uid", "subject", "from", "to", "cc", "bcc", "messageid", "listid", "body", "expungeheader", "" }; @@ -25,9 +28,8 @@ struct xapian_fts_backend { struct fts_backend backend; - char * path; + char * path = NULL; long partial,full; - bool attachments; char * guid; char * boxname; @@ -45,12 +47,16 @@ long perf_nb; long perf_uid; long perf_dt; + + long nb_pushes; + long max_push; }; struct xapian_fts_backend_update_context { struct fts_backend_update_context ctx; char * tbi_field=NULL; + bool isattachment=false; bool tbi_isfield; uint32_t tbi_uid=0; }; @@ -69,10 +75,10 @@ static int fts_backend_xapian_init(struct fts_backend *_backend, const char **error_r) { - if(verbose>0) i_info("fts_backend_xapian_init"); + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; + + if(verbose>0) i_info("fts_backend_xapian_init : %s",_backend->name); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)_backend; const char *const *tmp, *env; long len; @@ -83,11 +89,13 @@ backend->path = NULL; backend->old_guid = NULL; backend->old_boxname = NULL; - backend->attachments = false; verbose = 0; backend->partial = 0; backend->full = 0; + backend->nb_pushes=0; + backend->max_push=0; + env = mail_user_plugin_getenv(_backend->ns->user, "fts_xapian"); if (env == NULL) { @@ -114,7 +122,7 @@ } else if (strncmp(*tmp,"attachments=",12)==0) { - if(atol(*tmp + 12)>0) backend->attachments=true; + // Legacy } else { @@ -144,28 +152,16 @@ return -1; } - const char * path = mailbox_list_get_root_forced(_backend->ns->list, MAILBOX_LIST_PATH_TYPE_INDEX); - backend->path = i_strconcat(path, "/" XAPIAN_FILE_PREFIX, NULL); - - struct stat sb; - if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) - { - if (mailbox_list_mkdir_root(backend->backend.ns->list, backend->path, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) - { - i_error("FTS Xapian: can not create '%s'",backend->path); - return -1; - } - } + if(fts_backend_xapian_set_path(backend)<0) return -1; - if(verbose>0) i_info("FTS Xapian: Starting with partial=%ld full=%ld attachments=%d verbose=%d",backend->partial,backend->full,backend->attachments,verbose); + if(verbose>0) i_info("FTS Xapian: Starting with partial=%ld full=%ld verbose=%d",backend->partial,backend->full,verbose); return 0; } static void fts_backend_xapian_deinit(struct fts_backend *_backend) { - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)_backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; if(verbose>0) i_info("FTS Xapian: Deinit %s)",backend->path); @@ -188,8 +184,7 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_get_last_uid"); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)_backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; *last_uid_r = 0; @@ -242,18 +237,12 @@ static int fts_backend_xapian_update_deinit(struct fts_backend_update_context *_ctx) { - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)ctx->ctx.backend; + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_update_deinit (%s)",backend->path); - struct timeval tp; - gettimeofday(&tp, NULL); - long current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; - - fts_backend_xapian_release(backend,"update_deinit", current_time); + fts_backend_xapian_release(backend,"update_deinit",0); i_free(ctx); @@ -264,10 +253,8 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_update_set_mailbox"); - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)ctx->ctx.backend; + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; fts_backend_xapian_set_box(backend, box); } @@ -276,10 +263,8 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_update_expunge"); - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)ctx->ctx.backend; + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; if(!fts_backend_xapian_check_access(backend)) { @@ -332,11 +317,9 @@ { if(verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_set_build_key"); - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *)ctx->ctx.backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; ctx->tbi_isfield=false; ctx->tbi_uid=0; @@ -364,9 +347,7 @@ if((backend->perf_nb - backend->perf_pt)>=200) { backend->perf_pt = backend->perf_nb; - struct timeval tp; - gettimeofday(&tp, NULL); - long dt = tp.tv_sec * 1000 + tp.tv_usec / 1000 - backend->perf_dt; + long dt = fts_backend_xapian_current_time() - backend->perf_dt; double r=0; if(dt>0) { @@ -384,19 +365,27 @@ if(verbose>1) i_info("FTS Xapian: New part (Header=%s,Type=%s,Disposition=%s)",field,type,disposition); // Verify content-type - if((type != NULL) && (strncmp(type,"text",4)!=0)) + + if(key->type == FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY) { - if(verbose>0) i_info("FTS Xapian: Skipping part of type '%s'",type); + if(verbose>0) i_info("FTS Xapian: Skipping binary part of type '%s'",type); return FALSE; } - // Verify content-disposition - if((disposition != NULL) && (!backend->attachments) && ((strstr(disposition,"filename=")!=NULL) || (strstr(disposition,"attachment")!=NULL))) + if((type != NULL) && (strncmp(type,"text",4)!=0) && ((disposition==NULL) || ((strstr(disposition,"filename=")==NULL) && (strstr(disposition,"attachment")==NULL)))) { - if(verbose>0) i_info("FTS Xapian: Skipping part of type '%s' and disposition '%s'",type,disposition); + if(verbose>0) i_info("FTS Xapian: Non-binary & non-text part of type '%s'",type); return FALSE; } + // Verify content-disposition + ctx->isattachment=false; + if((disposition != NULL) && ((strstr(disposition,"filename=")!=NULL) || (strstr(disposition,"attachment")!=NULL))) + { + if(verbose>0) i_info("FTS Xapian: Found part as attachment of type '%s' and disposition '%s'",type,disposition); + ctx->isattachment=true; + } + // Fill-in field if(field==NULL) { @@ -422,7 +411,7 @@ } if(i>=HDRS_NB) { - if(verbose>1) i_info("FTS Xapian: Unknown header (indexing) '%s'",ctx->tbi_field); + if(verbose>1) i_info("FTS Xapian: Unknown header '%s' of part",ctx->tbi_field); i_free(ctx->tbi_field); ctx->tbi_field=NULL; return FALSE; @@ -447,10 +436,9 @@ static void fts_backend_xapian_update_unset_build_key(struct fts_backend_update_context *_ctx) { - if(verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_unset_build_key"); + if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_update_unset_build_key"); - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; if(ctx->tbi_field!=NULL) { @@ -464,26 +452,32 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_refresh"); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) _backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; - struct timeval tp; - gettimeofday(&tp, NULL); - long current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; - - fts_backend_xapian_release(backend,"refresh", current_time); + fts_backend_xapian_release(backend,"refresh", 0); return 0; } static int fts_backend_xapian_update_build_more(struct fts_backend_update_context *_ctx, const unsigned char *data, size_t size) { - if(verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_build_more"); + struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *) ctx->ctx.backend; - struct xapian_fts_backend_update_context *ctx = - (struct xapian_fts_backend_update_context *)_ctx; - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) ctx->ctx.backend; + if(verbose>1) + { + if(ctx->isattachment) + { + char * t = i_strdup("NODATA"); + if(data != NULL) { i_free(t); t = i_strndup(data,40); } + i_info("FTS Xapian: Indexing part as attachment (data like '%s')",t); + i_free(t); + } + else + { + i_info("FTS Xapian: Indexing part as text"); + } + } if(ctx->tbi_uid<1) return 0; @@ -499,13 +493,10 @@ return -1; } - struct timeval tp; - gettimeofday(&tp, NULL); - long current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; - - if(!fts_backend_xapian_test_memory()) + if(!fts_backend_xapian_test_memory(backend,d2.length())) { - fts_backend_xapian_release(backend,"Low memory indexing", current_time); + if(verbose>0) i_info("FTS Xapian: Warning Low memory"); + fts_backend_xapian_release(backend,"Low memory indexing", 0); if(!fts_backend_xapian_check_access(backend)) { i_error("FTS Xapian: Buildmore: Can not open db (2)"); @@ -518,16 +509,41 @@ if(ctx->tbi_isfield) { ok=fts_backend_xapian_index_hdr(backend,ctx->tbi_uid,ctx->tbi_field, &d2); + if(!ok) + { + if(verbose>0) i_info("FTS Xapian: Flushing memory and retrying"); + fts_backend_xapian_release(backend,"Flushing memory indexing hdr", 0); + if(fts_backend_xapian_check_access(backend)) + { + ok=fts_backend_xapian_index_hdr(backend,ctx->tbi_uid,ctx->tbi_field, &d2); + } + else + { + i_error("FTS Xapian: Buildmore: Can not open db (3)"); + } + } } else { ok=fts_backend_xapian_index_text(backend,ctx->tbi_uid,ctx->tbi_field, &d2); + if(!ok) + { + if(verbose>0) i_info("FTS Xapian: Flushing memory and retrying"); + fts_backend_xapian_release(backend,"Flushing memory indexing text", 0); + if(fts_backend_xapian_check_access(backend)) + { + ok=fts_backend_xapian_index_text(backend,ctx->tbi_uid,ctx->tbi_field, &d2); + } + else + { + i_error("FTS Xapian: Buildmore: Can not open db (4)"); + } + } } backend->commit_updates++; - gettimeofday(&tp, NULL); - current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; + long current_time = fts_backend_xapian_current_time(); if( (!ok) || (backend->commit_updates>XAPIAN_COMMIT_ENTRIES) || ((current_time - backend->commit_time) > XAPIAN_COMMIT_TIMEOUT*1000) ) { @@ -541,8 +557,7 @@ static int fts_backend_xapian_optimize(struct fts_backend *_backend) { - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) _backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; i_info("FTS Xapian: fts_backend_xapian_optimize '%s'",backend->path); @@ -580,8 +595,7 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_rescan"); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) _backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; struct stat sb; if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) @@ -631,16 +645,11 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_lookup"); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) _backend; + struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; - if(fts_backend_xapian_set_box(backend, box)<0) - return -1; + if(fts_backend_xapian_set_box(backend, box)<0) return -1; - /* Performance calc */ - struct timeval tp; - gettimeofday(&tp, NULL); - long current_time = tp.tv_sec * 1000 + tp.tv_usec / 1000; + long current_time = fts_backend_xapian_current_time(); Xapian::Database * dbr; @@ -671,6 +680,7 @@ XResultSet * r=fts_backend_xapian_query(dbr,qs); long n=r->size; + if(verbose>0) { i_info("FTS Xapian: QUery '%s' -> %ld results",qs->get_string().c_str(),n); } i_array_init(&(result->definite_uids),r->size); @@ -696,9 +706,7 @@ /* Performance calc */ if(verbose>0) { - gettimeofday(&tp, NULL); - long dt = tp.tv_sec * 1000 + tp.tv_usec / 1000 - current_time; - i_info("FTS Xapian: %ld results in %ld ms",n,dt); + i_info("FTS Xapian: %ld results in %ld ms",n,fts_backend_xapian_current_time() - current_time); } return 0; } @@ -707,10 +715,8 @@ { if(verbose>0) i_info("FTS Xapian: fts_backend_xapian_lookup_multi"); - struct xapian_fts_backend *backend = - (struct xapian_fts_backend *) _backend; - ARRAY(struct fts_result) box_results; + struct fts_result *box_result; int i; @@ -719,12 +725,22 @@ { box_result = array_append_space(&box_results); box_result->box = boxes[i]; - if(fts_backend_xapian_lookup(_backend, boxes[i], args, flags, box_result)<1) return -1; + if(fts_backend_xapian_lookup(_backend, boxes[i], args, flags, box_result)<0) + { + void* p=&box_results; + p_free(result->pool, p); + return -1; + } } + + array_append_zero(&box_results); + result->box_results = array_idx_modifiable(&box_results, 0); + return 0; } -struct fts_backend fts_backend_xapian = { +struct fts_backend fts_backend_xapian = +{ .name = "xapian", .flags = FTS_BACKEND_FLAG_BUILD_FULL_WORDS, .v = { @@ -748,4 +764,3 @@ NULL } }; - diff -Nru -w dovecot-fts-xapian-1.4.7/src/fts-xapian-plugin.c dovecot-fts-xapian-1.4.9a/src/fts-xapian-plugin.c --- dovecot-fts-xapian-1.4.7/src/fts-xapian-plugin.c 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/src/fts-xapian-plugin.c 2021-04-24 16:27:55.000000000 -0400 @@ -7,13 +7,11 @@ void fts_xapian_plugin_init(struct module *module ATTR_UNUSED) { - //i_warning("fts_xapian_plugin_init"); fts_backend_register(&fts_backend_xapian); } void fts_xapian_plugin_deinit(void) { - //i_warning("fts_xapian_plugin_deinit"); fts_backend_unregister(fts_backend_xapian.name); } diff -Nru -w dovecot-fts-xapian-1.4.7/src/fts-xapian-plugin.h dovecot-fts-xapian-1.4.9a/src/fts-xapian-plugin.h --- dovecot-fts-xapian-1.4.7/src/fts-xapian-plugin.h 2021-01-31 14:06:29.000000000 -0500 +++ dovecot-fts-xapian-1.4.9a/src/fts-xapian-plugin.h 2021-04-24 16:27:55.000000000 -0400 @@ -5,13 +5,12 @@ #define FTS_XAPIAN_PLUGIN_H #include "lib.h" -#include "mail-storage-private.h" -#include "mailbox-list-private.h" -#include "mail-search.h" +#include "fts-api-private.h" #include "fts-api.h" -#include "module-context.h" +#include "mail-search.h" +#include "mail-storage-private.h" #include "mail-user.h" -#include "fts-api-private.h" +#include "module-context.h" #include "restrict-process-size.h" extern const char *fts_xapian_plugin_dependencies[];
