Hi all, Please find attached a patch against current Mutt tip to add LMDB backend support. I must stress that this patch wasn't written by me, I've made sure it applies to current Mutt and works correctly.
The performance aspect of LMDB over BerkelyDB is significant, and in the light of licence aspects LMDB gets preferred by some distros these days. FWIW, the patch is part of Mutt on Gentoo. Please consider including the attached patch for the next Mutt release. I'll try to address any issues people might raise, if any. Thanks, Fabian -- Fabian Groffen Gentoo on a different level
Add LMDB backend support for header cache Pietro Cerutti <[email protected]> Derived on the original from JP Mens: https://gist.github.com/jpmens/15969d9d678a3d450e4e The following performance patch was manually applied on top of the original patch: https://github.com/neomutt/neomutt/commit/7e5380cd4c40d119ff83b2cf5f51f2cdb8a95ab3 Related ticked: https://dev.mutt.org/trac/ticket/3691 diff -r ae421b91c441 configure.ac --- a/configure.ac Wed Jan 25 14:15:58 2017 +0100 +++ b/configure.ac Wed Jan 25 14:43:14 2017 +0100 @@ -915,6 +915,7 @@ AC_ARG_WITH(tokyocabinet, AS_HELP_STRING AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if it is available])) AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available])) AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available])) +AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available])) db_found=no if test x$enable_hcache = xyes @@ -959,6 +960,15 @@ then db_requested=bdb fi fi + if test -n "$with_lmdb" && test "$with_lmdb" != "no" + then + if test "$db_requested" != "auto" + then + AC_MSG_ERROR([more than one header cache engine requested.]) + else + db_requested=lmdb + fi + fi dnl -- Tokyo Cabinet -- if test "$with_tokyocabinet" != "no" \ @@ -1046,7 +1056,8 @@ then dnl -- BDB -- ac_bdb_prefix="$with_bdb" - if test x$ac_bdb_prefix != xno && test $db_found = no + if test x$with_bdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = bdb then if test x$ac_bdb_prefix = xyes || test x$ac_bdb_prefix = x then @@ -1102,6 +1113,34 @@ then fi fi + dnl -- LMDB -- + if test x$with_lmdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = lmdb + then + if test "$with_lmdb" != "yes" + then + CPPFLAGS="$CPPFLAGS -I$with_lmdb/include" + LDFLAGS="$LDFLAGS -L$with_lmdb/lib" + fi + saved_LIBS="$LIBS" + LIBS="$LIBS -llmdb" + AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[ + ac_cv_mdbenvcreate=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <lmdb.h>]], [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[]) + ]) + LIBS="$saved_LIBS" + if test "$ac_cv_mdbenvcreate" = yes + then + AC_DEFINE(HAVE_LMDB, 1, [LMDB Support]) + MUTTLIBS="$MUTTLIBS -llmdb" + db_found=lmdb + fi + if test "$db_requested" != auto && test "$db_found" != "$db_requested" + then + AC_MSG_ERROR([LMDB could not be used. Check config.log for details.]) + fi + fi + if test $db_found = no then AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache]) diff -r ae421b91c441 hcache.c --- a/hcache.c Wed Jan 25 14:15:58 2017 +0100 +++ b/hcache.c Wed Jan 25 14:43:14 2017 +0100 @@ -32,6 +32,9 @@ #include <gdbm.h> #elif HAVE_DB4 #include <db.h> +#elif HAVE_LMDB +#define LMDB_DB_SIZE (1024 * 1024 * 1024) +#include <lmdb.h> #endif #include <errno.h> @@ -83,6 +86,61 @@ struct header_cache static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len); static void mutt_hcache_dbt_empty_init(DBT * dbt); +#elif HAVE_LMDB +enum mdb_txn_mode +{ + txn_uninitialized = 0, + txn_read = 1 << 0, + txn_write = 1 << 1 +}; +struct header_cache +{ + MDB_env *env; + MDB_txn *txn; + MDB_dbi db; + char *folder; + unsigned int crc; + enum mdb_txn_mode txn_mode; +}; + +static int mdb_get_r_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode & (txn_read | txn_write)) > 0) + return MDB_SUCCESS; + + if (h->txn) + rc = mdb_txn_renew(h->txn); + else + rc = mdb_txn_begin(h->env, NULL, MDB_RDONLY, &h->txn); + + if (rc == MDB_SUCCESS) + h->txn_mode = txn_read; + + return rc; +} + +static int mdb_get_w_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode == txn_write)) + return MDB_SUCCESS; + + if (h->txn) + { + if (h->txn_mode == txn_read) + mdb_txn_reset(h->txn); + h->txn = NULL; + } + + rc = mdb_txn_begin(h->env, NULL, 0, &h->txn); + if (rc == MDB_SUCCESS) + h->txn_mode = txn_write; + + return rc; +} #endif typedef union @@ -744,6 +802,11 @@ mutt_hcache_fetch_raw (header_cache_t *h #elif HAVE_DB4 DBT key; DBT data; +#elif HAVE_LMDB + MDB_val key; + MDB_val data; + size_t folderlen; + int rc; #endif if (!h) @@ -760,6 +823,37 @@ mutt_hcache_fetch_raw (header_cache_t *h h->db->get(h->db, NULL, &key, &data, 0); return data.data; +#elif HAVE_LMDB + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + + folderlen = strlen(h->folder); + ksize = folderlen + keylen(path + folderlen); + key.mv_data = (char *)path; + key.mv_size = ksize; + data.mv_data = NULL; + data.mv_size = 0; + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + h->txn = NULL; + fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc)); + return NULL; + } + rc = mdb_get(h->txn, h->db, &key, &data); + if (rc == MDB_NOTFOUND) { + return NULL; + } + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_get: %s\n", mdb_strerror(rc)); + return NULL; + } + /* Caller frees the data we return, so I MUST make a copy of it */ + + char *d = safe_malloc(data.mv_size); + memcpy(d, data.mv_data, data.mv_size); + + return d; + #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -825,6 +919,11 @@ mutt_hcache_store_raw (header_cache_t* h #elif HAVE_DB4 DBT key; DBT databuf; +#elif HAVE_LMDB + MDB_val key; + MDB_val databuf; + size_t folderlen; + int rc; #endif if (!h) @@ -843,6 +942,30 @@ mutt_hcache_store_raw (header_cache_t* h databuf.ulen = dlen; return h->db->put(h->db, NULL, &key, &databuf, 0); +#elif HAVE_LMDB + folderlen = strlen(h->folder); + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + ksize = folderlen + keylen(path + folderlen); + + key.mv_data = (char *)path; + key.mv_size = ksize; + databuf.mv_data = data; + databuf.mv_size = dlen; + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_put(h->txn, h->db, &key, &databuf, 0); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + return rc; + } + return rc; #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -1146,6 +1269,106 @@ mutt_hcache_delete(header_cache_t *h, co mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); return h->db->del(h->db, NULL, &key, 0); } +#elif HAVE_LMDB + +static int +hcache_open_lmdb (struct header_cache* h, const char* path) +{ + int rc; + + h->txn = NULL; + + rc = mdb_env_create(&h->env); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_create: %s", mdb_strerror(rc)); + return -1; + } + + mdb_env_set_mapsize(h->env, LMDB_DB_SIZE); + + rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_open: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_txn_begin: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_dbi_open: %s", mdb_strerror(rc)); + goto fail_dbi; + } + + mdb_txn_reset(h->txn); + h->txn_mode = txn_uninitialized; + return 0; + +fail_dbi: + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + +fail_env: + mdb_env_close(h->env); + return -1; +} + +void +mutt_hcache_close(header_cache_t *h) +{ + if (!h) + return; + + if (h->txn && h->txn_mode == txn_write) + { + mdb_txn_commit(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + + mdb_env_close(h->env); + FREE (&h->folder); + FREE (&h); +} + +int +mutt_hcache_delete(header_cache_t *h, const char *filename, + size_t(*keylen) (const char *fn)) +{ + MDB_val key; + int rc; + + if (!h) + return -1; + + if (filename[0] == '/') + filename++; + + key.mv_data = (char *)filename; + key.mv_size = strlen(filename); + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_del(h->txn, h->db, &key, NULL); + if (rc != MDB_SUCCESS) { + if (rc != MDB_NOTFOUND) { + fprintf(stderr, "mdb_del: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + return rc; + } + + return rc; +} #endif header_cache_t * @@ -1163,6 +1386,8 @@ mutt_hcache_open(const char *path, const hcache_open = hcache_open_gdbm; #elif HAVE_DB4 hcache_open = hcache_open_db4; +#elif HAVE_LMDB + hcache_open = hcache_open_lmdb; #endif /* Calculate the current hcache version from dynamic configuration */ @@ -1200,7 +1425,11 @@ mutt_hcache_open(const char *path, const hcachever = digest.intval; } +#if HAVE_LMDB + h->db = 0; +#else h->db = NULL; +#endif h->folder = get_foldername(folder); h->crc = hcachever; @@ -1235,6 +1464,11 @@ const char *mutt_hcache_backend (void) { return DB_VERSION_STRING; } +#elif HAVE_LMDB +const char *mutt_hcache_backend (void) +{ + return "lmdb " MDB_VERSION_STRING; +} #elif HAVE_GDBM const char *mutt_hcache_backend (void) {
signature.asc
Description: Digital signature
