[PATCH 02/10] lib/n_d_add_message: refactor test for new/ghost messages

[David Bremner]

The switch is easier to understand than the side effects in the if
test. It also potentially allows us more flexibility in breaking up
this function into smaller pieces, since passing private_status around
is icky.
---
 lib/add-message.cc | 23 +--
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 5fe2c45b..0f09415e 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -570,7 +570,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 notmuch_message_t *message = NULL;
 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
 notmuch_private_status_t private_status;
-notmuch_bool_t is_ghost = false;
+notmuch_bool_t is_ghost = FALSE, is_new = FALSE;
 
 const char *date, *header;
 const char *from, *to, *subject;
@@ -655,7 +655,17 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
talloc_free (message_id);
 
-   if (message == NULL) {
+   /* We cannot call notmuch_message_get_flag for a new message */
+   switch (private_status) {
+   case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+   is_ghost = FALSE;
+   is_new = TRUE;
+   break;
+   case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+   is_ghost = notmuch_message_get_flag (message, 
NOTMUCH_MESSAGE_FLAG_GHOST);
+   is_new = FALSE;
+   break;
+   default:
ret = COERCE_STATUS (private_status,
 "Unexpected status value from 
_notmuch_message_create_for_message_id");
goto DONE;
@@ -663,18 +673,11 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
_notmuch_message_add_filename (message, filename);
 
-   /* Is this a newly created message object or a ghost
-* message?  We have to be slightly careful: if this is a
-* blank message, it's not safe to call
-* notmuch_message_get_flag yet. */
-   if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND ||
-   (is_ghost = notmuch_message_get_flag (
-   message, NOTMUCH_MESSAGE_FLAG_GHOST))) {
+   if (is_new || is_ghost) {
_notmuch_message_add_term (message, "type", "mail");
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-
ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
if (ret)
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 01/10] lib: isolate n_d_add_message and helper functions into own file

[David Bremner]

'database.cc' is becoming a monster, and it's hard to follow what the
various static functions are used for. It turns out that about 1/3 of
this file notmuch_database_add_message and helper functions not used
by any other function. This commit isolates this code into it's own
file.

Some side effects of this refactoring:

- find_doc_ids becomes the non-static (but still private)
  _notmuch_database_find_doc_ids
- a few instances of 'string' have 'std::' prepended, avoiding the
  need for 'using namespace std;' in the new file.
---
 lib/Makefile.local |   1 +
 lib/add-message.cc | 721 
 lib/database-private.h |   8 +
 lib/database.cc| 732 +
 4 files changed, 737 insertions(+), 725 deletions(-)
 create mode 100644 lib/add-message.cc

diff --git a/lib/Makefile.local b/lib/Makefile.local
index d36fd5a0..e29fb081 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -48,6 +48,7 @@ libnotmuch_cxx_srcs = \
$(dir)/directory.cc \
$(dir)/index.cc \
$(dir)/message.cc   \
+   $(dir)/add-message.cc   \
$(dir)/message-property.cc \
$(dir)/query.cc \
$(dir)/query-fp.cc  \
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index ..5fe2c45b
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,721 @@
+#include "database-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } else if (*s == ')') {
+   nesting--;
+   } else if (*s == '\\') {
+   if (*(s+1))
+   s++;
+   }
+   s++;
+   }
+   }
+}
+
+*str = s;
+}
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+static char *
+_parse_message_id (void *ctx, const char *message_id, const char **next)
+{
+const char *s, *end;
+char *result;
+
+if (message_id == NULL || *message_id == '\0')
+   return NULL;
+
+s = message_id;
+
+skip_space_and_comments ();
+
+/* Skip any unstructured text as well. */
+while (*s && *s != '<')
+   s++;
+
+if (*s == '<') {
+   s++;
+} else {
+   if (next)
+   *next = s;
+   return NULL;
+}
+
+skip_space_and_comments ();
+
+end = s;
+while (*end && *end != '>')
+   end++;
+if (next) {
+   if (*end)
+   *next = end + 1;
+   else
+   *next = end;
+}
+
+if (end > s && *end == '>')
+   end--;
+if (end <= s)
+   return NULL;
+
+result = talloc_strndup (ctx, s, end - s + 1);
+
+/* Finally, collapse any whitespace that is within the message-id
+ * itself. */
+{
+   char *r;
+   int len;
+
+   for (r = result, len = strlen (r); *r; r++, len--)
+   if (*r == ' ' || *r == '\t')
+   memmove (r, r+1, len);
+}
+
+return result;
+}
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+ const char *message_id,
+ GHashTable *hash,
+ const char *refs)
+{
+char *ref, *last_ref = NULL;
+
+if (refs == NULL || *refs == '\0')
+   return NULL;
+
+while (*refs) {
+   ref = _parse_message_id (ctx, refs, );
+
+   if (ref && strcmp (ref, message_id)) {
+   g_hash_table_add (hash, ref);
+   last_ref = ref;
+   }
+}
+
+/* The return value of this function is used to add a parent
+ * 

Re: [PATCH 1/7] test: add known broken test for indexing html

[David Bremner]

David Bremner  writes:

> 'quite' on IRC reported that notmuch new was grinding to a halt during
> initial indexing, and we eventually narrowed the problem down to some
> html parts with large embedded images. These cause the number of terms
> added to the Xapian database to explode (the first 400 messages
> generated 4.6M unique terms), and of course the resulting terms are
> not much use for searching.
>
> The second test is sanity check for any "improved" indexing of HTML.

pushed the first patch in the series to master.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

notmuch-0.24.1: missing header include

[Thomas Klausner]

Hi!

On Solaris, notmuch-0.24.1 does not compile because lib/message.cc
uses index(3) but does not include strings.h.

Please apply the attached patch or a similar one.

Thanks,
 Thomas
$NetBSD: patch-lib_notmuch-private.h,v 1.1 2017/04/20 09:06:34 jperkin Exp $

Include strings.h for index(3).

--- lib/notmuch-private.h.orig  2017-04-01 12:29:38.0 +
+++ lib/notmuch-private.h
@@ -38,6 +38,7 @@ NOTMUCH_BEGIN_DECLS
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: notmuch-emacs and Emacs26 (master)

[Steven Allen]

This appears to have been fixed by commit
89898e43c7ceef28bb3c2116b4d8a3ec96d9c8da in Emacs master.


signature.asc
Description: PGP signature
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: how to configure notmuch-emacs to automatically show application/pgp partsFcc: sent -unread

[David Bremner]

David Bremner  writes:

> Gregor Zattler  writes:
>
>> Dear notmuch users and developers, could one please enlighten me,
>> how to configure notmuch-emacs in order to automatically show
>> application/pgp mime parts in emails as in e.g. cryptograhically
>> signed debian security advisories (as attached).
>
> I think currently there is no support for application/pgp content
> time. The MIME RFC (from 2001!) describes that as "since withdrawn").
> I don't know much more about it, maybe a later RFC revived this mime
> type.

By the way, I looked at that message in the debian list archives, and it
is not using application/pgp. So I guess some mail server is "helpfully"
repackaging things on receipt.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

error building against xapian 1.3.4

[Matthew Lear]

Hi. On a new machine that I've been setting up (running Ubuntu 16.04 LTS),
I'm unable to build notmuch from git with HEAD @

* 77c9ec1 -  (HEAD -> master, origin/master, origin/HEAD) test: add known
broken test for indexing html (2017-04-20 06:59:40 -0300) 

The error looks like this:
...
CXX  -g -O2 lib/database.o
CXX  -g -O2 lib/parse-time-vrp.o
CXX  -g -O2 lib/directory.o
CXX  -g -O2 lib/index.o
CXX  -g -O2 lib/message.o
CXX  -g -O2 lib/message-property.o
CXX  -g -O2 lib/query.o
CXX  -g -O2 lib/query-fp.o
CXX  -g -O2 lib/config.o
CXX  -g -O2 lib/regexp-fields.o
lib/regexp-fields.cc: In member function ‘virtual Xapian::Query
RegexpFieldProcessor::operator()(const string&)’:
lib/regexp-fields.cc:159:38: error: ‘class RegexpPostingSource’ has no
member named ‘release’
  return Xapian::Query (postings->release ());
  ^
make: *** [lib/regexp-fields.o] Error 1

Even though xapian 1.4 is marked as stable, Ubuntu 16.04 has xapian 1.3.4
installed. I'd rather not start hacking around updating xapian and
dependant packages since the whole point of me having this machine is to
use it as a stable baseline with other colleagues who I work with.

Here's the output from the configure checks:

Sanity checking C compilation environment... OK.
Sanity checking C++ compilation environment... OK.
Reading libnotmuch version from source... OK.
Checking for Xapian development files... Yes (1.3.4).
Checking for Xapian compaction support... Yes.
Checking for Xapian FieldProcessor API... Yes.
Checking for Xapian lock retry support... Yes.
Testing default Xapian backend... chert
Checking for GMime development files... Yes.
Checking for Glib development files (>= 2.22)... Yes.
Checking for zlib (>= 1.2.5.2)... Yes.
Checking for talloc development files... Yes.
Checking for python... Yes (python).
Checking for valgrind development files... No (but that's fine).
Checking for bash-completion (>= 1.90)... Yes.
Checking if emacs is available... Yes.
Checking if doxygen is available... No (so will not install api docs)
Checking for ruby development files... No (skipping ruby bindings)
Checking if sphinx is available and supports nroff output... No (so will
not install man pages).
Checking if desktop-file-install is available... Yes.
Checking which platform we are on... Linux
Checking for /usr/local/lib in ldconfig... Yes
Checking for canonicalize_file_name... Yes.
Checking for getline... Yes.
Checking for strcasestr... Yes.
Checking for strsep... Yes.
Checking for timegm... Yes.
Checking for dirent.d_type... Yes.
Checking for standard version of getpwuid_r... Yes.
Checking for standard version of asctime_r... Yes.
Checking for rpath support... Yes.
Checking for -Wl,--as-needed... Yes.
Checking for -Wl,--no-undefined... Yes.
Checking for available C++ compiler warning flags...
-Wall -Wextra -Wwrite-strings
Checking for available C compiler warning flags...
-Wall -Wextra -Wwrite-strings -Wmissing-declarations

All required packages were found. You may now run the following
...

regexp-fields.cc was created in commit:

* 55524bb -  lib: regexp matching in 'subject' and 'from' (2017-03-03
17:46:48 -0400) 

If I revert to the previous commit:

* | 31b8ce4 -  lib: create field processors from prefix table (2017-03-03
07:15:13 -0400) 

I can build ok.

Is anybody still using xapian 1.3.x ? I was really hoping to keep up to
date with notmuch on my Ubuntu machine...

Cheers,
--  Matt
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/4] build: do not export compat functions from lib

[Jani Nikula]

Commits 9db214527213 ("lib/gen-version-script.h: add getline and
getdelim to notmuch.sym if needed") and 3242e29e57ac ("build: add
canonicalize_file_name to symbols exported from libnotmuch.so")
started exporting compat functions from libnotmuch so that the cli
could use them. But we shouldn't export such functions from the
library. They are not part of our ABI. Instead, the cli should include
its own copies of the compat functions.
---
 Makefile.local| 1 +
 lib/gen-version-script.sh | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile.local b/Makefile.local
index 3d3474e0c97e..6bc78ef8e969 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -212,6 +212,7 @@ dataclean: distclean
rm -rf $(DATACLEAN)
 
 notmuch_client_srcs =  \
+   $(notmuch_compat_srcs)  \
command-line-arguments.c\
debugger.c  \
status.c\
diff --git a/lib/gen-version-script.sh b/lib/gen-version-script.sh
index 5621f2a9fd85..c98a07b0c701 100644
--- a/lib/gen-version-script.sh
+++ b/lib/gen-version-script.sh
@@ -24,6 +24,5 @@ while read sym; do
;;
 esac
 done
-nm $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $2 == "T" && $3 ~ 
"^(getline|getdelim|canonicalize_file_name)$" {print $3 ";"}'
 sed  -n 's/^[[:space:]]*\(notmuch_[a-z_]*\)[[:space:]]*(.*/ \1;/p' $HEADER
 printf "local: *;\n};\n"
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

RFC on default notmuch-show mime part buttons

[William Casarin]

I recently had to do this in my notmuch config to remove a lot of button
noise in the notmuch-show buffer:

  (defun jb55/notmuch-show-insert-header-p (part hide)
;; Show all part buttons except for text/plain and multipart
(let ((mime-type (notmuch-show-mime-type part)))
  (not (member mime-type (list "multipart/alternative"
   "multipart/mixed"
   "text/plain")

  (setq notmuch-show-insert-header-p-function 
'jb55/notmuch-show-insert-header-p)

where the default is:

  (defun notmuch-show-insert-header-p (part hide)
;; Show all part buttons except for the first part if it is text/plain.
(let ((mime-type (notmuch-show-mime-type part)))
  (not (and (string= mime-type "text/plain")
  (<= (plist-get part :id) 1)

Is there any reason this be a bad default? The multipart buttons
don't seem to be much use AFAICT, and they litter almost all messages.

Thanks!

--
https://jb55.com
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 03/10] lib: factor out message-id parsing to separate file.

[David Bremner]

This is really pure C string parsing, and doesn't need to be mixed in
with the Xapian/C++ layer. Although not strictly necessary, it also
makes it a bit more natural to call _parse_message_id from multiple
compilation units.
---
 lib/Makefile.local|   1 +
 lib/add-message.cc| 106 +-
 lib/message-id.c  |  96 +
 lib/notmuch-private.h |  14 +++
 4 files changed, 112 insertions(+), 105 deletions(-)
 create mode 100644 lib/message-id.c

diff --git a/lib/Makefile.local b/lib/Makefile.local
index e29fb081..643199ad 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -36,6 +36,7 @@ libnotmuch_c_srcs =   \
$(dir)/filenames.c  \
$(dir)/string-list.c\
$(dir)/message-file.c   \
+   $(dir)/message-id.c \
$(dir)/messages.c   \
$(dir)/sha1.c   \
$(dir)/built-with.c \
diff --git a/lib/add-message.cc b/lib/add-message.cc
index 0f09415e..314016a8 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -1,109 +1,5 @@
 #include "database-private.h"
 
-/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
- * a (potentially nested) parenthesized sequence with '\' used to
- * escape any character (including parentheses).
- *
- * If the sequence to be skipped continues to the end of the string,
- * then 'str' will be left pointing at the final terminating '\0'
- * character.
- */
-static void
-skip_space_and_comments (const char **str)
-{
-const char *s;
-
-s = *str;
-while (*s && (isspace (*s) || *s == '(')) {
-   while (*s && isspace (*s))
-   s++;
-   if (*s == '(') {
-   int nesting = 1;
-   s++;
-   while (*s && nesting) {
-   if (*s == '(') {
-   nesting++;
-   } else if (*s == ')') {
-   nesting--;
-   } else if (*s == '\\') {
-   if (*(s+1))
-   s++;
-   }
-   s++;
-   }
-   }
-}
-
-*str = s;
-}
-
-/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
- * comments, and the '<' and '>' delimiters.
- *
- * If not NULL, then *next will be made to point to the first character
- * not parsed, (possibly pointing to the final '\0' terminator.
- *
- * Returns a newly talloc'ed string belonging to 'ctx'.
- *
- * Returns NULL if there is any error parsing the message-id. */
-static char *
-_parse_message_id (void *ctx, const char *message_id, const char **next)
-{
-const char *s, *end;
-char *result;
-
-if (message_id == NULL || *message_id == '\0')
-   return NULL;
-
-s = message_id;
-
-skip_space_and_comments ();
-
-/* Skip any unstructured text as well. */
-while (*s && *s != '<')
-   s++;
-
-if (*s == '<') {
-   s++;
-} else {
-   if (next)
-   *next = s;
-   return NULL;
-}
-
-skip_space_and_comments ();
-
-end = s;
-while (*end && *end != '>')
-   end++;
-if (next) {
-   if (*end)
-   *next = end + 1;
-   else
-   *next = end;
-}
-
-if (end > s && *end == '>')
-   end--;
-if (end <= s)
-   return NULL;
-
-result = talloc_strndup (ctx, s, end - s + 1);
-
-/* Finally, collapse any whitespace that is within the message-id
- * itself. */
-{
-   char *r;
-   int len;
-
-   for (r = result, len = strlen (r); *r; r++, len--)
-   if (*r == ' ' || *r == '\t')
-   memmove (r, r+1, len);
-}
-
-return result;
-}
-
 /* Parse a References header value, putting a (talloc'ed under 'ctx')
  * copy of each referenced message-id into 'hash'.
  *
@@ -126,7 +22,7 @@ parse_references (void *ctx,
return NULL;
 
 while (*refs) {
-   ref = _parse_message_id (ctx, refs, );
+   ref = _notmuch_message_id_parse (ctx, refs, );
 
if (ref && strcmp (ref, message_id)) {
g_hash_table_add (hash, ref);
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index ..d7541d50
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,96 @@
+#include "notmuch-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } 

[PATCH 06/10] lib: index message files with duplicate message-ids

[David Bremner]

The corresponding xapian document just gets more terms added to it,
but this doesn't seem to break anything. Values on the other hand get
overwritten, which is a bit annoying, but arguably it is not worse to
take the values (from, subject, date) from the last file indexed
rather than the first.
---
 lib/add-message.cc | 20 +++-
 test/T160-json.sh  |  4 ++--
 test/T670-duplicate-mid.sh |  2 --
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 2922eaa9..ae9b14a7 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -529,19 +529,21 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-   ret = _notmuch_database_link_message (notmuch, message,
+   }
+
+   ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
-   if (ret)
-   goto DONE;
+   if (ret)
+   goto DONE;
 
-   _notmuch_message_set_header_values (message, date, from, subject);
+   _notmuch_message_set_header_values (message, date, from, subject);
 
-   ret = _notmuch_message_index_file (message, message_file);
-   if (ret)
-   goto DONE;
-   } else {
+   ret = _notmuch_message_index_file (message, message_file);
+   if (ret)
+   goto DONE;
+
+   if (! is_new && !is_ghost)
ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-   }
 
_notmuch_message_sync (message);
 } catch (const Xapian::Error ) {
diff --git a/test/T160-json.sh b/test/T160-json.sh
index ac51895e..07955a2b 100755
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -71,8 +71,8 @@ test_begin_subtest "Format version: too high"
 test_expect_code 21 "notmuch search --format-version=999 \\*"
 
 test_begin_subtest "Show message: multiple filenames"
-add_message "[id]=message...@example.com [filename]=copy1"
-add_message "[id]=message...@example.com [filename]=copy2"
+add_message '[id]=message...@example.com [filename]=copy1 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
+add_message '[id]=message...@example.com [filename]=copy2 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
 cat < EXPECTED
 [
 [
diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh
index ced28a21..f1952555 100755
--- a/test/T670-duplicate-mid.sh
+++ b/test/T670-duplicate-mid.sh
@@ -6,7 +6,6 @@ add_message '[id]="duplicate"' '[subject]="message 1" 
[filename]=copy1'
 add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2'
 
 test_begin_subtest 'Search for second subject'
-test_subtest_known_broken
 cat 

[PATCH 2/4] compat: don't include compat.h from the feature test source

[Jani Nikula]

The feature test code should test the build environment, and none of
the compat code should interfere with that. Don't include compat.h
from the feature test source. There should be no functional changes
here, but this is just the right thing to do.
---
 compat/have_timegm.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/compat/have_timegm.c b/compat/have_timegm.c
index b62b7937feab..483fc3b6685d 100644
--- a/compat/have_timegm.c
+++ b/compat/have_timegm.c
@@ -1,5 +1,4 @@
 #include 
-#include "compat.h"
 
 int main()
 {
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: how to configure notmuch-emacs to automatically show application/pgp partsFcc: sent -unread

[David Bremner]

Gregor Zattler  writes:

> Dear notmuch users and developers, could one please enlighten me,
> how to configure notmuch-emacs in order to automatically show
> application/pgp mime parts in emails as in e.g. cryptograhically
> signed debian security advisories (as attached).

I think currently there is no support for application/pgp content
time. The MIME RFC (from 2001!) describes that as "since withdrawn").
I don't know much more about it, maybe a later RFC revived this mime
type.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 0/4] build fixes and improvements

[Jani Nikula]

I was looking into the Meson build system [1], and tried converting
notmuch to use it as a learning experience. I'm not sure if that'll lead
anywhere, but I noted some issues in the current build while at it.

The first two are genuine fixes, the last two I think make the build
easier to understand, but also make it more straightforward to convert
the build to Meson.

BR,
Jani.


Jani Nikula (4):
  build: do not export compat functions from lib
  compat: don't include compat.h from the feature test source
  build: switch to hiding libnotmuch symbols by default
  build: visibility=default for library structs is no longer needed

 .gitignore |  1 -
 Makefile.local |  1 +
 compat/have_timegm.c   |  1 -
 lib/Makefile.local | 25 -
 lib/database-private.h |  4 
 lib/gen-version-script.sh  | 29 -
 lib/message.cc |  2 +-
 lib/notmuch-private.h  | 14 ++
 lib/notmuch.h  |  4 
 lib/notmuch.sym|  7 +++
 lib/query.cc   |  2 +-
 lib/thread.cc  |  2 +-
 test/T360-symbol-hiding.sh |  2 +-
 13 files changed, 30 insertions(+), 64 deletions(-)
 delete mode 100644 lib/gen-version-script.sh
 create mode 100644 lib/notmuch.sym

-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Support for limiting number of displayed search results in Emacs saved searches ?

[Olivier Berger]

Hi.

I'd like to be able to add the --limit option to the notmuch search
command spawned from saved searches in Emacs notmuch-hello.

But the construction of the search string only supports search terms and
not options, apparently.

Any hope ?

Many thanks in advance.
-- 
Olivier BERGER 
http://www-public.telecom-sudparis.eu/~berger_o/ - OpenPGP-Id: 2048R/5819D7E8
Ingenieur Recherche - Dept INF
Institut Mines-Telecom, Telecom SudParis, Evry (France)
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[RFC][PATCH] notmuch-search: add file size search feature

[Ioan-Adrian Ratiu]

I need to keep track of my maildir file sizes and for a while the
parsers/scripts which I wrote were problematic, especially for viewing
& interacting with large collections of email.

Since I started using notmuch, having a way to filter my email search
results based on file sizes would prove handy, together with notmuch's
incremental indexing/parsing, that would really be helpful.

So this commit adds the following capability (sizes are expressed in
bytes because that's what I'm interested in, other inputs can be added
further on). Replace the range arg to get mails of a specific size:

notmuch search --output=files --sort=biggest-first  filesize:1000..10 | 
xargs du -s

I didn't think of another front-end for this other than creating
saved searches from the Emacs GUI, which work well enough for me.

One thing I'm not sure how to handle in code is duplicate files
for a message ID (only the MsgID determines whether files are
"duplicates", right?). Any ideas how best to store/handle duplicates?

Signed-off-by: Ioan-Adrian Ratiu 
---
 lib/database-private.h |  1 +
 lib/database.cc|  6 ++
 lib/index.cc   | 10 ++
 lib/message-file.c | 18 +-
 lib/message.cc | 29 +
 lib/notmuch-private.h  | 16 
 lib/notmuch.h  | 14 ++
 lib/query.cc   |  6 ++
 notmuch-search.c   |  2 ++
 9 files changed, 101 insertions(+), 1 deletion(-)

diff --git a/lib/database-private.h b/lib/database-private.h
index ab3d9691..a7e0a020 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -217,6 +217,7 @@ struct _notmuch_database {
 Xapian::ValueRangeProcessor *value_range_processor;
 Xapian::ValueRangeProcessor *date_range_processor;
 Xapian::ValueRangeProcessor *last_mod_range_processor;
+Xapian::ValueRangeProcessor *filesize_range_processor;
 };
 
 /* Prior to database version 3, features were implied by the database
diff --git a/lib/database.cc b/lib/database.cc
index 5bc131a3..e6d5dd11 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -290,6 +290,7 @@ prefix_t prefix_table[] = {
 { "subject",   "XSUBJECT", NOTMUCH_FIELD_EXTERNAL |
NOTMUCH_FIELD_PROBABILISTIC |
NOTMUCH_FIELD_PROCESSOR},
+{ "filesize",  "XFILESIZE",NOTMUCH_FIELD_EXTERNAL },
 };
 
 static void
@@ -1076,6 +1077,7 @@ notmuch_database_open_verbose (const char *path,
notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor 
(NOTMUCH_VALUE_TIMESTAMP);
notmuch->date_range_processor = new ParseTimeValueRangeProcessor 
(NOTMUCH_VALUE_TIMESTAMP);
notmuch->last_mod_range_processor = new 
Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
+   notmuch->filesize_range_processor = new 
Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_FILESIZE, "filesize:");
 
notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
notmuch->query_parser->set_database (*notmuch->xapian_db);
@@ -1084,6 +1086,7 @@ notmuch_database_open_verbose (const char *path,
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->value_range_processor);
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->date_range_processor);
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->last_mod_range_processor);
+   notmuch->query_parser->add_valuerangeprocessor 
(notmuch->filesize_range_processor);
 
for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
const prefix_t *prefix = _table[i];
@@ -1160,6 +1163,8 @@ notmuch_database_close (notmuch_database_t *notmuch)
 notmuch->date_range_processor = NULL;
 delete notmuch->last_mod_range_processor;
 notmuch->last_mod_range_processor = NULL;
+delete notmuch->filesize_range_processor;
+notmuch->filesize_range_processor = NULL;
 
 return status;
 }
@@ -2557,6 +2562,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
}
 
_notmuch_message_add_filename (message, filename);
+   _notmuch_message_add_filesize (message, message_file);
 
/* Is this a newly created message object or a ghost
 * message?  We have to be slightly careful: if this is a
diff --git a/lib/index.cc b/lib/index.cc
index 8c145540..e8655bc1 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -441,6 +441,8 @@ _notmuch_message_index_file (notmuch_message_t *message,
 InternetAddressList *addresses;
 const char *from, *subject;
 notmuch_status_t status;
+unsigned long filesize;
+char *filesize_str;
 
 status = _notmuch_message_file_get_mime_message (message_file,
 _message);
@@ -464,6 +466,14 @@ _notmuch_message_index_file (notmuch_message_t *message,
 subject = g_mime_message_get_subject (mime_message);
 

Announcing Astroid v0.9

[Gaute Hope]

Greetings,

Astroid v0.9 has been released!

 Astroid is a lightweight and fast graphical threads-with-tags email
 client for notmuch. Written in C++ using GTK+, WebKit and gmime.

Astroid can be acquired at:

$ git clone https://github.com/astroidmail/astroid.git

 Usage and instructions:

   https://github.com/astroidmail/astroid#astroid-mua

 once you get Astroid running press '?' to get a list of keybindings
 for the current context. For 'The Tour' check out:

   https://github.com/astroidmail/astroid/wiki


( P.S: We now also have a homebrew tap for Mac OS users: 
https://github.com/astroidmail/homebrew-astroid !)


## Changes since v0.8

* Archive or close thread and then directly open next or next unread thread
 in originating thread-index.
* Drop use of old deprecated non _st functions, use their new names. Will
 require 0.24 or newer.
* No longer support pre-lastmod notmuches.
* Catch missing content errors for messages, not fail.
* Wait and block R/O database opening, not fail.
* Defer changed_thread events on thread-index while still loading.
* Do not update tags in thread-view when not ready.
* Configuration option thread_view.expand_flagged determines whether flagged
 messages are expanded by default.
* Better handle poll thread. Make sure to clean up connections here and in 
send_message.
* Polling can be cancelled with C-c in main window.


Regards, Gaute



pgpVm2osPKhan.pgp
Description: PGP signature
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 3/3] emacs: notmuch-search: add filesize sorting

[Ioan-Adrian Ratiu]

Besides the previous date-based result orderings (oldest-first and
newest-first) add two more filesize-based orderings: biggest-first
smallest-first.

The orderings are interchangeable, you can specify any one as the
default via notmuch-search-default-sort-order or as the preffered
ordering for a saved search (via the :sort-order property).

The only restriction I've imposed is that you can't toggle from a
date-based result ordering to a filesize-based one. Meaning you can
toggle as before from oldest-first to newest-first or from the new
smallest-first to biggest-first and viceversa, but you can't toggle
from oldest-first to biggest-first. I've found this kind of toggling
confusing so I've prohibited it, but enabling it is trivial if need be.

Signed-off-by: Ioan-Adrian Ratiu 
---
 emacs/notmuch-hello.el |  9 ++---
 emacs/notmuch-lib.el   |  4 +++-
 emacs/notmuch.el   | 12 ++--
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index 3ba2a16b..51117577 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -95,7 +95,9 @@ searches so they still work in customize."
(choice :tag " Sort Order"
(const :tag "Default" nil)
(const :tag "Oldest-first" oldest-first)
-   (const :tag "Newest-first" newest-first)))
+   (const :tag "Newest-first" newest-first)
+   (const :tag "Biggest-first" biggest-first)
+   (const :tag "Smallest-first" 
smallest-first)))
 (group :format "%v" :inline t (const :format "" 
:search-type)
(choice :tag " Search Type"
(const :tag "Search mode" nil)
@@ -120,8 +122,9 @@ a plist. Supported properties are
shown. If not present then the :query property
is used.
   :sort-order  Specify the sort order to be used for the search.
-   Possible values are 'oldest-first 'newest-first or
-   nil. Nil means use the default sort order.
+   Possible values are 'oldest-first 'newest-first
+   'biggest-first 'smallest-first or nil.
+   Nil means use the default sort order.
   :search-type Specify whether to run the search in search-mode
or tree mode. Set to 'tree to specify tree
mode, set to nil (or anything except tree) to
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 34ffa712..ded75c2e 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -100,7 +100,9 @@ search results. Note that any filtered searches created by
 `notmuch-search-filter' retain the search order of the parent
 search."
   :type '(choice (const :tag "oldest-first" oldest-first)
-(const :tag "newest-first" newest-first))
+(const :tag "newest-first" newest-first)
+(const :tag "biggest-first" biggest-first)
+(const :tag "smallest-first" smallest-first))
   :group 'notmuch-search)
 
 (defcustom notmuch-poll-script nil
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index 248b97c7..4ba29880 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -1023,12 +1023,20 @@ same relative position within the new buffer."
   "Toggle the current search order.
 
 This command toggles the sort order for the current search. The
-default sort order is defined by `notmuch-search-default-sort-order'."
+default sort order is defined by `notmuch-search-default-sort-order'.
+
+There are two types of orderings: by date and by filesize. Toggling is
+permitted only within a specific type (for ex. you can't toggle from
+oldest-first to smallest-first, only from oldest-first to newest-first)
+to avoid confusion."
   (interactive)
   (setq notmuch-search-default-sort-order
(case notmuch-search-default-sort-order
  ('oldest-first 'newest-first)
- (otherwise 'oldest-first)))
+ ('newest-first 'oldest-first)
+ ('biggest-first 'smallest-first)
+ ('smallest-first 'biggest-first)
+ (otherwise notmuch-search-default-sort-order)))
   (notmuch-search-refresh-view))
 
 (defun notmuch-group-disjunctive-query-string (query-string)
-- 
2.12.2

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/3] emacs: make notmuch-search-oldest-first generic

[Ioan-Adrian Ratiu]

The current search result order logic assumes results are always
sorted by date and thus uses a boolean switch for oldest/newest
ordering specifications.

This is problematic if I want to introduce other result orderings,
like for example based on the mail-file size with smallest/biggest
ordering specifications.

In the interest of keeping all current logic intact and reusable,
while at the same time supporting multiple search result orderings,
change the defcustom configuration notmuch-search-oldest-first to
notmuch-search-default-sort-order which takes values 'oldest-first
and 'newest-first (for now).

Implementing new result orderings thus becomes a simple matter of
adding more possible entries for notmuch-search-default-sort-order.

Aside from the UI variable rename change, this commit should be
totally transparent for the user, it does not modify or add any new
result sorting logic.

Signed-off-by: Ioan-Adrian Ratiu 
---
 doc/notmuch-emacs.rst  |  4 ++--
 emacs/notmuch-hello.el | 15 +++
 emacs/notmuch-jump.el  | 11 +--
 emacs/notmuch-lib.el   |  7 ---
 emacs/notmuch-tree.el  |  2 +-
 emacs/notmuch.el   | 29 +++--
 6 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst
index 5e25996f..66a69bb8 100644
--- a/doc/notmuch-emacs.rst
+++ b/doc/notmuch-emacs.rst
@@ -169,8 +169,8 @@ variables.
 Control how each thread of messages is presented in the
 ``notmuch-show-mode`` buffer
 
-:index:`notmuch-search-oldest-first`
-Display the oldest threads at the top of the buffer
+:index:`notmuch-search-default-sort-order`
+Control the default search result method
 
 .. _notmuch-show:
 
diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index c858a20b..3ba2a16b 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -28,7 +28,7 @@
 (require 'notmuch-lib)
 (require 'notmuch-mua)
 
-(declare-function notmuch-search "notmuch" ( query oldest-first 
target-thread target-line continuation))
+(declare-function notmuch-search "notmuch" ( query sort-order 
target-thread target-line continuation))
 (declare-function notmuch-poll "notmuch" ())
 (declare-function notmuch-tree "notmuch-tree"
   ( query query-context target buffer-name 
open-target))
@@ -381,7 +381,7 @@ afterwards.")
 (setq search (notmuch-hello-trim search))
 (let ((history-delete-duplicates t))
   (add-to-history 'notmuch-search-history search)))
-  (notmuch-search search notmuch-search-oldest-first))
+  (notmuch-search search notmuch-search-default-sort-order))
 
 (defun notmuch-hello-add-saved-search (widget)
   (interactive)
@@ -443,7 +443,7 @@ diagonal."
 (notmuch-search (widget-get widget
:notmuch-search-terms)
(widget-get widget
-   :notmuch-search-oldest-first
+   :notmuch-search-sort-order
 
 (defun notmuch-saved-search-count (search)
   (car (process-lines notmuch-command "count" search)))
@@ -575,10 +575,9 @@ with `notmuch-hello-query-counts'."
  (widget-insert (make-string column-indent ? )))
  (let* ((name (plist-get elem :name))
 (query (plist-get elem :query))
-(oldest-first (case (plist-get elem :sort-order)
-(newest-first nil)
-(oldest-first t)
-(otherwise notmuch-search-oldest-first)))
+(sort-order (if (plist-get elem :sort-order)
+(plist-get elem :sort-order)
+  notmuch-search-default-sort-order))
 (search-type (eq (plist-get elem :search-type) 'tree))
 (msg-count (plist-get elem :count)))
(widget-insert (format "%8s "
@@ -586,7 +585,7 @@ with `notmuch-hello-query-counts'."
(widget-create 'push-button
   :notify #'notmuch-hello-widget-search
   :notmuch-search-terms query
-  :notmuch-search-oldest-first oldest-first
+  :notmuch-search-sort-order sort-order
   :notmuch-search-type search-type
   name)
(setq column-indent
diff --git a/emacs/notmuch-jump.el b/emacs/notmuch-jump.el
index 3e20b8c7..716d39b8 100644
--- a/emacs/notmuch-jump.el
+++ b/emacs/notmuch-jump.el
@@ -50,15 +50,14 @@ fast way to jump to a saved search from anywhere in 
Notmuch."
(when key
  (let ((name (plist-get saved-search :name))
(query (plist-get saved-search :query))
-   (oldest-first
-(case (plist-get saved-search :sort-order)
-  (newest-first nil)
-  (oldest-first t)

Limiting number of results in saved searches in Emacs ?

[Olivier Berger]

Hi.

I can't find a way to limit the number of results in saved searches in
Emacs. Mainly to avoid OOM issues.

It seems that --limit=N could help on the command-line.

However, the construction of the saved searches only allow to type search terms
and no options like --limit.

Would anyone have a better suggestion ?

Thanks in advance.

Best regards,
-- 
Olivier BERGER 
http://www-public.telecom-sudparis.eu/~berger_o/ - OpenPGP-Id: 2048R/5819D7E8
Ingenieur Recherche - Dept INF
Institut Mines-Telecom, Telecom SudParis, Evry (France)

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/3] notmuch-search: implement 'filesize' search

[Ioan-Adrian Ratiu]

I need to keep track of my mail file sizes and writing custom
parsers/scripts and running them on the maildir is problematic
especially for managing large collections of email.

Having a way to search email based on file sizes would be very useful,
also copled with notmuch's incremental maildir indexing/parsing, rapid
result navigation/sorting via Emacs makes my life a lot easier.

So this commit adds the following capability (sizes are in bytes
because that's what I'm interested in, other inputs can be added
I guess). Replace the range param to get mails of a specific size:

notmuch search --sort=biggest-first filesize:1000..10

I didn't think of another front-end for this other than the command
line and saved searches from the Emacs GUI, which work well enough for
me with this patch series, but it would be simple to also expose the
filesize info on the screen somewhere in the results/show buffer.

Another interesting idea would be to offer some optional statistics
in the UI buffers like we do for saved searches `query-count` to show
stuff like disk space occupied by mail files maching a search, etc.

Signed-off-by: Ioan-Adrian Ratiu 
---
 lib/database-private.h |  1 +
 lib/database.cc|  6 ++
 lib/index.cc   | 10 ++
 lib/message-file.c | 18 +-
 lib/message.cc | 29 +
 lib/notmuch-private.h  | 16 
 lib/notmuch.h  | 14 ++
 lib/query.cc   |  6 ++
 notmuch-search.c   |  2 ++
 9 files changed, 101 insertions(+), 1 deletion(-)

diff --git a/lib/database-private.h b/lib/database-private.h
index ab3d9691..a7e0a020 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -217,6 +217,7 @@ struct _notmuch_database {
 Xapian::ValueRangeProcessor *value_range_processor;
 Xapian::ValueRangeProcessor *date_range_processor;
 Xapian::ValueRangeProcessor *last_mod_range_processor;
+Xapian::ValueRangeProcessor *filesize_range_processor;
 };
 
 /* Prior to database version 3, features were implied by the database
diff --git a/lib/database.cc b/lib/database.cc
index 5bc131a3..e6d5dd11 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -290,6 +290,7 @@ prefix_t prefix_table[] = {
 { "subject",   "XSUBJECT", NOTMUCH_FIELD_EXTERNAL |
NOTMUCH_FIELD_PROBABILISTIC |
NOTMUCH_FIELD_PROCESSOR},
+{ "filesize",  "XFILESIZE",NOTMUCH_FIELD_EXTERNAL },
 };
 
 static void
@@ -1076,6 +1077,7 @@ notmuch_database_open_verbose (const char *path,
notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor 
(NOTMUCH_VALUE_TIMESTAMP);
notmuch->date_range_processor = new ParseTimeValueRangeProcessor 
(NOTMUCH_VALUE_TIMESTAMP);
notmuch->last_mod_range_processor = new 
Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
+   notmuch->filesize_range_processor = new 
Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_FILESIZE, "filesize:");
 
notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
notmuch->query_parser->set_database (*notmuch->xapian_db);
@@ -1084,6 +1086,7 @@ notmuch_database_open_verbose (const char *path,
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->value_range_processor);
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->date_range_processor);
notmuch->query_parser->add_valuerangeprocessor 
(notmuch->last_mod_range_processor);
+   notmuch->query_parser->add_valuerangeprocessor 
(notmuch->filesize_range_processor);
 
for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
const prefix_t *prefix = _table[i];
@@ -1160,6 +1163,8 @@ notmuch_database_close (notmuch_database_t *notmuch)
 notmuch->date_range_processor = NULL;
 delete notmuch->last_mod_range_processor;
 notmuch->last_mod_range_processor = NULL;
+delete notmuch->filesize_range_processor;
+notmuch->filesize_range_processor = NULL;
 
 return status;
 }
@@ -2557,6 +2562,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
}
 
_notmuch_message_add_filename (message, filename);
+   _notmuch_message_add_filesize (message, message_file);
 
/* Is this a newly created message object or a ghost
 * message?  We have to be slightly careful: if this is a
diff --git a/lib/index.cc b/lib/index.cc
index 8c145540..e8655bc1 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -441,6 +441,8 @@ _notmuch_message_index_file (notmuch_message_t *message,
 InternetAddressList *addresses;
 const char *from, *subject;
 notmuch_status_t status;
+unsigned long filesize;
+char *filesize_str;
 
 status = _notmuch_message_file_get_mime_message (message_file,
 _message);
@@ -464,6 +466,14 @@ 

Re: [PATCH] configure: Be more verbose when compiler sanity checks fail

[David Bremner]

Tomi Ollila  writes:

> By printing the executed command line and re-executing it without
> output redirection user gets better information how to proceed
> (or ask for help) to resolve this problem.

pushed to master (some time ago)

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH 2/2] emacs: make modifications to message Fcc vars buffer-local

[David Bremner]

David Bremner  writes:

> Tomi Ollila  writes:
>
>>
>> One thing come to my mind: would it be possible to use buffer-local
>> variable message-fcc-handler-function in notmuch-message-mode buffers so
>> that such wrapper functions defined below would not be needed. i.e.
>>
>>   (make-local-variable 'message-fcc-handler-function)
>>   (setq message-fcc-handler-function #'notmuch-fcc-handler)
>>
>
> This was my first approach, and I actually filed
>
>  http://debbugs.gnu.org/cgi/bugreport.cgi?bug=21174
>
> about why it doesn't work.
>
> Great minds think alike / Fools seldom differ, as they say ;)
>
> d

For the record, this bug is fixed in emacs git.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: some issues with emacs 25

[Matthew Lear]

On 4 Mar 2017 11:03, "David Bremner"  wrote:

David Bremner  writes:

> Matthew Lear  writes:
>
>
>> Thanks David. Yes it does. After recompiling the v25 lisp with these
>> changes, I'm unable to reproduce the problems with both the test emails I
>> sent you. Wonderful :-)
>> Are you going to raise this with upstream?
>> Cheers,
>>  Matt
>
> Yes, I've filed
>
>  https://debbugs.gnu.org/cgi/bugreport.cgi?bug=25828
>

That bug should be fixed in emacs master branch.

d


It was indeed fixed...that is until I updated my emacs git tree to 25.2.1.
I've given up composing emails in new frames as it's now broken again -
this time when replying to emails which have quoted/embedded html in the
body. Feels like two steps forward, one step back :-(
--  Matt
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: Fwd: error building against xapian 1.3.4

[David Bremner]

Matthew Lear  writes:
>
> Even though xapian 1.4 is marked as stable, Ubuntu 16.04 has xapian 1.3.4
> installed. I'd rather not start hacking around updating xapian and
> dependant packages since the whole point of me having this machine is to
> use it as a stable baseline with other colleagues who I work with.
>

16.04 also has xapian 1.2.22, so I'd suggest using that. You will miss
out some newer features, but it should build fine.
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

ANNOUNCE: muchsync 3 released

[David Mazieres]

Muchsync 3 is now available from the usual place:

http://www.muchsync.org/

Muchsync synchronizes mail and tags in your notmuch database across
machines.  Not much has changed since the last release, because the tool
already worked well.  However, there are two changes people requested:

  * Muchsync only buffers 128MiB of data, rather than an unbounded
amount.  People synchronizing large databases on machines without
enough swap space were running out of memory.  A 128MiB buffer
should still be plenty to saturate the network.

  * There's a new option --newid to change the local replica id.  This
potentially allows a faster way of initializing muchsync:  Copying
your entire mail database and .notmuch-config file from the original
machine, then running "muchsync --newid" on the destination machine
to give the new replica a unique identity.

Enjoy.

David
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] Replace index(3) with strchr(3)

[David Bremner]

Fredrik Fornwall  writes:

> The index(3) function has been deprecated in POSIX since 2001 and
> removed in 2008, and most code in notmuch already calls strchr(3).
>

pushed to master (some time ago)

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] Makefile.local: have all files in release tarball be owned by root

[David Bremner]

Tomi Ollila  writes:

> The tar content `git archive` creates (reproducibly) have owner and
> group set to 'root'. (GNU) tar writes user ids to the added file
> `version` by default. The contents of tar archive looks better and
> more consistent when owner and group in all files are the same.
>
> While at it, split this long command line to multiple lines.

pushed to master (some time ago)

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] emacs/notmuch.el: don't use 'function' as variable name.

[David Bremner]

David Bremner  writes:

> In principle this should work, but at least in some pre-release
> versions of emacs26, this causes problems inside lexical let [1].
>
> [1]: https://debbugs.gnu.org/cgi/bugreport.cgi?bug=26406

pushed to master (some time ago). In the meantime, also fixed upstream
for the next major release of emacs.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 4/6] lib/index: generalize filter name

[David Bremner]

We can't very well call it uuencode if it is going to filter other
things as well.
---
 lib/index.cc | 92 +++-
 1 file changed, 48 insertions(+), 44 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 02b35b81..3bb1ac1c 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -26,8 +26,8 @@
 
 /* Oh, how I wish that gobject didn't require so much noisy boilerplate!
  * (Though I have at least eliminated some of the stock set...) */
-typedef struct _NotmuchFilterDiscardUuencode NotmuchFilterDiscardUuencode;
-typedef struct _NotmuchFilterDiscardUuencodeClass 
NotmuchFilterDiscardUuencodeClass;
+typedef struct _NotmuchFilterDiscardNonTerms NotmuchFilterDiscardNonTerms;
+typedef struct _NotmuchFilterDiscardNonTermsClass 
NotmuchFilterDiscardNonTermsClass;
 
 typedef void (*filter_fun) (GMimeFilter *filter, char *in, size_t len, size_t 
prespace,
char **out, size_t *outlen, size_t *outprespace);
@@ -41,44 +41,29 @@ typedef struct {
 } scanner_state_t;
 
 /**
- * NotmuchFilterDiscardUuencode:
+ * NotmuchFilterDiscardNonTerms:
  *
  * @parent_object: parent #GMimeFilter
  * @encode: encoding vs decoding
  * @state: State of the parser
  *
- * A filter to discard uuencoded portions of an email.
- *
- * A uuencoded portion is identified as beginning with a line
- * matching:
- *
- * begin [0-7][0-7][0-7] .*
- *
- * After that detection, and beginning with the following line,
- * characters will be discarded as long as the first character of each
- * line begins with M and subsequent characters on the line are within
- * the range of ASCII characters from ' ' to '`'.
- *
- * This is not a perfect UUencode filter. It's possible to have a
- * message that will legitimately match that pattern, (so that some
- * legitimate content is discarded). And for most UUencoded files, the
- * final line of encoded data (the line not starting with M) will be
- * indexed.
+ * A filter to discard non terms portions of an email, i.e. stuff not
+ * worth indexing.
  **/
-struct _NotmuchFilterDiscardUuencode {
+struct _NotmuchFilterDiscardNonTerms {
 GMimeFilter parent_object;
 GMimeContentType *content_type;
 filter_fun real_filter;
 int state;
 };
 
-struct _NotmuchFilterDiscardUuencodeClass {
+struct _NotmuchFilterDiscardNonTermsClass {
 GMimeFilterClass parent_class;
 };
 
-static GMimeFilter *notmuch_filter_discard_uuencode_new (GMimeContentType 
*content);
+static GMimeFilter *notmuch_filter_discard_non_terms_new (GMimeContentType 
*content);
 
-static void notmuch_filter_discard_uuencode_finalize (GObject *object);
+static void notmuch_filter_discard_non_terms_finalize (GObject *object);
 
 static GMimeFilter *filter_copy (GMimeFilter *filter);
 static void filter_filter (GMimeFilter *filter, char *in, size_t len, size_t 
prespace,
@@ -91,14 +76,14 @@ static void filter_reset (GMimeFilter *filter);
 static GMimeFilterClass *parent_class = NULL;
 
 static void
-notmuch_filter_discard_uuencode_class_init (NotmuchFilterDiscardUuencodeClass 
*klass)
+notmuch_filter_discard_non_terms_class_init (NotmuchFilterDiscardNonTermsClass 
*klass)
 {
 GObjectClass *object_class = G_OBJECT_CLASS (klass);
 GMimeFilterClass *filter_class = GMIME_FILTER_CLASS (klass);
 
 parent_class = (GMimeFilterClass *) g_type_class_ref (GMIME_TYPE_FILTER);
 
-object_class->finalize = notmuch_filter_discard_uuencode_finalize;
+object_class->finalize = notmuch_filter_discard_non_terms_finalize;
 
 filter_class->copy = filter_copy;
 filter_class->filter = filter_filter;
@@ -107,7 +92,7 @@ notmuch_filter_discard_uuencode_class_init 
(NotmuchFilterDiscardUuencodeClass *k
 }
 
 static void
-notmuch_filter_discard_uuencode_finalize (GObject *object)
+notmuch_filter_discard_non_terms_finalize (GObject *object)
 {
 G_OBJECT_CLASS (parent_class)->finalize (object);
 }
@@ -115,15 +100,15 @@ notmuch_filter_discard_uuencode_finalize (GObject *object)
 static GMimeFilter *
 filter_copy (GMimeFilter *gmime_filter)
 {
-NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
+NotmuchFilterDiscardNonTerms *filter = (NotmuchFilterDiscardNonTerms *) 
gmime_filter;
 
-return notmuch_filter_discard_uuencode_new (filter->content_type);
+return notmuch_filter_discard_non_terms_new (filter->content_type);
 }
 
 static void
 filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t 
prespace,
   char **outbuf, size_t *outlen, size_t *outprespace) {
-NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
+NotmuchFilterDiscardNonTerms *filter = (NotmuchFilterDiscardNonTerms *) 
gmime_filter;
 (*filter->real_filter)(gmime_filter, inbuf, inlen, prespace, outbuf, 
outlen, outprespace);
 }
 
@@ -133,7 +118,7 @@ do_filter (const scanner_state_t states[],
   GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t 
prespace,

RE: Upcoming GMime 3.0 changes

[Daniel Kahn Gillmor]

On Tue 2017-05-09 10:37:30 -0300, David Bremner wrote:
> Just for the record, I have some patches in progress for porting to
> gmime-3.0. The main issue is the multiplicity of memory management
> models involved. I think the gmime 3.0 approach of using more stock glib
> memory management makes sense, but it will require a bit of work to make
> code that can compile against gmime-2.6 and gmime-3.0. I don't see us
> being able to drop support for gmime-2.6 for a few years, unfortunately.

out of curiosity, why do you think we won't be able to drop gmime-2.6
for a few years?  if it's due to the debian release cycle and wanting to
backport notmuch to stretch, i don't think i'd mind providing backports
of gmime 3.0 for that purpose.

   --dkg
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

RE: Upcoming GMime 3.0 changes

[David Bremner]

Jeffrey Stedfast  writes:

>> -Original Message-
>> From: David Bremner [mailto:da...@tethera.net]
>> Sent: Tuesday, March 14, 2017 9:18 AM
>> To: Jeffrey Stedfast ; notmuch@notmuchmail.org
>> Subject: Re: Upcoming GMime 3.0 changes
>> 
>> Jeffrey Stedfast  writes:
>> 
>> > Hello notmuch devs,
>> >
>> > I'm sending this email to inform you guys of some upcoming GMime 3.0
>> features and other changes that I've implemented.
>> >

Just for the record, I have some patches in progress for porting to
gmime-3.0. The main issue is the multiplicity of memory management
models involved. I think the gmime 3.0 approach of using more stock glib
memory management makes sense, but it will require a bit of work to make
code that can compile against gmime-2.6 and gmime-3.0. I don't see us
being able to drop support for gmime-2.6 for a few years, unfortunately.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 6/6] lib/index: add simple html filter

[David Bremner]

Just drop all (HTML) tags
---
 lib/index.cc   | 38 +-
 test/T680-html-indexing.sh |  5 -
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 1b420b75..217b58c0 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -211,6 +211,39 @@ filter_filter_uuencode (GMimeFilter *gmime_filter, char 
*inbuf, size_t inlen, si
 }
 
 static void
+filter_filter_html (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, 
size_t prespace,
+   char **outbuf, size_t *outlen, size_t *outprespace)
+{
+/* The following table is intended to implement this DFA (in 'dot'
+   format). Note that 2 and 3 are "hidden" states used to step through
+   the possible out edges of state 1.
+
+digraph html_filter {
+   0 -> 1  [label="<"];
+   0 -> 0;
+   1 -> 4 [label="'"];
+   1 -> 5 [label="\""];
+   1 -> 0 [label=">"];
+   1 -> 1;
+   4 -> 1 [label="'"];
+   4 -> 4;
+   5 -> 1 [label="\""];
+   5 -> 5;
+}
+*/
+static const scanner_state_t states[] = {
+   {0,  '<',  '<',  1,  0},
+   {1,  '\'', '\'', 4,  2},  /* scanning for quote or > */
+   {1,  '"',  '"',  5,  3},
+   {1,  '>',  '>',  0,  1},
+   {4,  '\'', '\'', 1,  4},  /* inside single quotes */
+   {5,  '"', '"',   1,  5},  /* inside double quotes */
+};
+do_filter(states, 1,
+ gmime_filter, inbuf, inlen, prespace, outbuf, outlen, 
outprespace);
+}
+
+static void
 filter_complete (GMimeFilter *filter, char *inbuf, size_t inlen, size_t 
prespace,
 char **outbuf, size_t *outlen, size_t *outprespace)
 {
@@ -257,7 +290,10 @@ notmuch_filter_discard_non_terms_new (GMimeContentType 
*content_type)
 filter = (NotmuchFilterDiscardNonTerms *) g_object_newv (type, 0, NULL);
 filter->state = 0;
 filter->content_type = content_type;
-filter->real_filter = filter_filter_uuencode;
+if (g_mime_content_type_is_type (content_type, "text", "html"))
+   filter->real_filter = filter_filter_html;
+else
+   filter->real_filter = filter_filter_uuencode;
 return (GMimeFilter *) filter;
 }
 
diff --git a/test/T680-html-indexing.sh b/test/T680-html-indexing.sh
index 5e9cc4cb..74f33708 100755
--- a/test/T680-html-indexing.sh
+++ b/test/T680-html-indexing.sh
@@ -5,10 +5,13 @@ test_description="indexing of html parts"
 add_email_corpus html
 
 test_begin_subtest 'embedded images should not be indexed'
-test_subtest_known_broken
 notmuch search kwpza7svrgjzqwi8fhb2msggwtxtwgqcxp4wbqr4wjddstqmeqa7 > OUTPUT
 test_expect_equal_file /dev/null OUTPUT
 
+test_begin_subtest 'ignore > in attribute text'
+notmuch search swordfish | notmuch_search_sanitize > OUTPUT
+test_expect_equal_file /dev/null OUTPUT
+
 test_begin_subtest 'non tag text should be indexed'
 notmuch search hunter2 | notmuch_search_sanitize > OUTPUT
 cat < EXPECTED
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

v2 of drop HTML tags when indexing

[David Bremner]

This obsoletes id:20170322112306.12060-1-da...@tethera.net

Compared to the previous version:

- drop test already pushed upstream

- add diagram of new state machine

- add longer, and hopefully more illuminating comment about how the
  (new) state machines are encoded

- don't create the variable 'current', stick with the existing 'next'
  as a state index/pointer
  
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/6] lib/index: Add another layer of indirection in filtering

[David Bremner]

We could add a second gmime filter subclass, but prefer to avoid
duplicating the boilerplate.
---
 lib/index.cc | 14 --
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 1c04cc3d..74a750b9 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -29,6 +29,8 @@
 typedef struct _NotmuchFilterDiscardUuencode NotmuchFilterDiscardUuencode;
 typedef struct _NotmuchFilterDiscardUuencodeClass 
NotmuchFilterDiscardUuencodeClass;
 
+typedef void (*filter_fun) (GMimeFilter *filter, char *in, size_t len, size_t 
prespace,
+   char **out, size_t *outlen, size_t *outprespace);
 /**
  * NotmuchFilterDiscardUuencode:
  *
@@ -57,6 +59,7 @@ typedef struct _NotmuchFilterDiscardUuencodeClass 
NotmuchFilterDiscardUuencodeCl
 struct _NotmuchFilterDiscardUuencode {
 GMimeFilter parent_object;
 GMimeContentType *content_type;
+filter_fun real_filter;
 int state;
 };
 
@@ -110,7 +113,14 @@ filter_copy (GMimeFilter *gmime_filter)
 
 static void
 filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t 
prespace,
-  char **outbuf, size_t *outlen, size_t *outprespace)
+  char **outbuf, size_t *outlen, size_t *outprespace) {
+NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
+(*filter->real_filter)(gmime_filter, inbuf, inlen, prespace, outbuf, 
outlen, outprespace);
+}
+
+static void
+filter_filter_uuencode (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, 
size_t prespace,
+   char **outbuf, size_t *outlen, size_t *outprespace)
 {
 NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
 register const char *inptr = inbuf;
@@ -223,7 +233,7 @@ notmuch_filter_discard_uuencode_new (GMimeContentType 
*content_type)
 filter = (NotmuchFilterDiscardUuencode *) g_object_newv (type, 0, NULL);
 filter->state = 0;
 filter->content_type = content_type;
-
+filter->real_filter = filter_filter_uuencode;
 return (GMimeFilter *) filter;
 }
 
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 5/6] lib/index.cc: generalize filter state machine

[David Bremner]

To match things more complicated than fixed strings, we need states
with multiple out arrows.
---
 lib/index.cc | 25 -
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 3bb1ac1c..1b420b75 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -129,16 +129,23 @@ do_filter (const scanner_state_t states[],
 g_mime_filter_set_size (gmime_filter, inlen, FALSE);
 outptr = gmime_filter->outbuf;
 
+next = filter->state;
 while (inptr < inend) {
-   if (*inptr >= states[filter->state].a &&
-   *inptr <= states[filter->state].b)
-   {
-   next = states[filter->state].next_if_match;
-   }
-   else
-   {
-   next = states[filter->state].next_if_not_match;
-   }
+/* Each state is defined by a contiguous set of rows of the
+* state table marked by a common value for '.state'. The
+* state numbers must be equal to the index of the first row
+* in a given state; thus the loop condition here looks for a
+* jump to a first row of a state, which is a real transition
+* in the underlying DFA.
+*/
+   do {
+   if (*inptr >= states[next].a && *inptr <= states[next].b)  {
+   next = states[next].next_if_match;
+   } else  {
+   next = states[next].next_if_not_match;
+   }
+
+   } while (next != states[next].state);
 
if (filter->state < first_skipping_state)
*outptr++ = *inptr;
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

RE: Upcoming GMime 3.0 changes

[Tomi Ollila]

On Tue, May 09 2017, David Bremner wrote:

> Daniel Kahn Gillmor  writes:
>
>>
>> out of curiosity, why do you think we won't be able to drop gmime-2.6
>> for a few years?  if it's due to the debian release cycle and wanting to
>> backport notmuch to stretch, i don't think i'd mind providing backports
>> of gmime 3.0 for that purpose.
>
> That was what I was thinking about (and other distros with similar
> release cycles). But even among our own developers I don't know we can
> just throw a switch and say "install gmime 3.0 before you git pull". 

I personally can compile gmime 3.0 for this (EOL'd) scientific linux 6.2
machine where I am writing this email, provided it does not need c/c++11
features(*). If gmime-3.0 could be statically linked to notmuch binary then
we could even provide an option to have it bundled into notmuch when
compiled (like I've included zlib...).

> I guess the other point is even with backports, there is still the issue
> of security support for older releases. We'll need to maintain the 2.6
> stuff in a branch, even if it only gets security/severe bug updates.

But these go to releases (how many old of those) which still have support
for gmime 2.6...

>
> Another question is how long Jeff plans to support 2.6 with bug fixes.
>
> d

Tomi

(*) I compiled Xapian 1.4 for Fedora 25; compiled fine and notmuch worked
with it. On SL6.2 compilation failed, and when running
/opt/rh/devtoolset-2/root/usr/bin/gcc compilation crashed at some point,

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

RE: Upcoming GMime 3.0 changes

[David Bremner]

Daniel Kahn Gillmor  writes:

>
> out of curiosity, why do you think we won't be able to drop gmime-2.6
> for a few years?  if it's due to the debian release cycle and wanting to
> backport notmuch to stretch, i don't think i'd mind providing backports
> of gmime 3.0 for that purpose.

That was what I was thinking about (and other distros with similar
release cycles). But even among our own developers I don't know we can
just throw a switch and say "install gmime 3.0 before you git pull".  I
guess the other point is even with backports, there is still the issue
of security support for older releases. We'll need to maintain the 2.6
stuff in a branch, even if it only gets security/severe bug updates.

Another question is how long Jeff plans to support 2.6 with bug fixes.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [notmuch] Re: v3 of regexp search for mid/folder/path

[David Bremner]

Tomi Ollila  writes:

> On Sun, May 07 2017, David Bremner wrote:
>>
>> Gauteh reported success with these patches on IRC. Anyone want more time
>> to review?
>
> Nope, but fix s/implimentation/implementation/ in 2/2 commit message :D

Done and pushed to master. Ispell provided my morning amusement by
suggesting I replace ui with Uzi.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[gmailieer] [release v0.1] Fast fetch and two-way tag synchronization between notmuch and GMail

[Gaute Hope]

Hi,

 gmailieer now has its first release!

'gmailieer' (or 'gmi') is a small program that can pull email and labels
from your GMail account and store them locally in a maildir with the
labels synchronized with a notmuch database. The changes to tags in the
notmuch database may be pushed back to your GMail account.

Main new features are:

* Exponential backoff on most remote requests making gmi more robust
 when synchronizing lots of changes.

* An API key is included so you can get started right away without getting
 any keys from Google.

Instructions and source code can be found here:

 https://github.com/gauteh/gmailieer

Disclaimer:
 This is still experimental, but gmi does not have access to delete
 e-mail on your remote account - only fetch and change labels, so damage
 should be limited.



 Thanks to several contributors for bug-fixes and debugging!


Regards, Gaute


pgpHm3cJHrObx.pgp
Description: PGP signature
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[gmailieer] [release v0.1] Fast fetch and two-way tag synchronization between notmuch and GMail

[Gaute Hope]

Hi,

 gmailieer now has its first release!

'gmailieer' (or 'gmi') is a small program that can pull email and labels
from your GMail account and store them locally in a maildir with the
labels synchronized with a notmuch database. The changes to tags in the
notmuch database may be pushed back to your GMail account.

Main new features are:

* Exponential backoff on most remote requests making gmi more robust
 when synchronizing lots of changes.

* An API key is included so you can get started right away without getting
 any keys from Google.

Instructions and source code can be found here:

 https://github.com/gauteh/gmailieer

Disclaimer:
 This is still experimental, but gmi does not have access to delete
 e-mail on your remote account - only fetch and change labels, so damage
 should be limited.



 Thanks to several contributors for bug-fixes and debugging!


Regards, Gaute
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/2] test: add known broken test --include=properties

[David Bremner]

Reported in [1], --include=properties currently implies
--include=tags, but it should not.

[1] id:87y3u8vjbo@tethera.net
---
 test/T610-message-property.sh | 9 +
 1 file changed, 9 insertions(+)

diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index 65ff19dc..7d95bde6 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -209,6 +209,15 @@ EOF
 notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
+test_begin_subtest "dump _only_ message properties"
+test_subtest_known_broken
+cat < EXPECTED
+#notmuch-dump batch-tag:3 properties
+#= 4efc743a.3060...@april.org 
fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice 
testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 
testkey3=bob3 testkey3=testvalue3
+EOF
+notmuch dump --include=properties > OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
 
 test_begin_subtest "restore missing message property (single line)"
 notmuch dump | grep '^#=' > BEFORE1
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Thunderbird import & parsing flight etickets

[Johannes Buchner]

Dear all,

I wrote a primitive script to copy emails from Thunderbird's storage
into notmuch, allowing one to use notmuch's power without having to
abandon Thunderbird. Perhaps you are interested in including it in the
import scripts on the notmuch website.
thunderbird-notmuch-import.py in
https://github.com/JohannesBuchner/flight-reservation-emails
This works with the help of mb2md, but finds the relevant mboxes
automatically. Maybe in the future it would be interesting to treat tags
"X-Mozilla-Keys: $label1" as notmuch tags.

I started that github project today to find emails that look like flight
etickets, and make a summary of the flights with their details. Google's
Android phones have a similar feature (flight ticket wallet), but there
is nothing for laptops as far as I can tell. I implemented one
convention, but it does not cover all flight etickets at the moment.
Hotel bookings may also be interesting. Feel free to fork and expand.

Cheers,
   Johannes
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 06/10] lib: index message files with duplicate message-ids

[David Bremner]

The corresponding xapian document just gets more terms added to it,
but this doesn't seem to break anything. Values on the other hand get
overwritten, which is a bit annoying, but arguably it is not worse to
take the values (from, subject, date) from the last file indexed
rather than the first.
---
 lib/add-message.cc | 20 +++-
 test/T160-json.sh  |  4 ++--
 test/T670-duplicate-mid.sh |  2 --
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 2922eaa9..ae9b14a7 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -529,19 +529,21 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-   ret = _notmuch_database_link_message (notmuch, message,
+   }
+
+   ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
-   if (ret)
-   goto DONE;
+   if (ret)
+   goto DONE;
 
-   _notmuch_message_set_header_values (message, date, from, subject);
+   _notmuch_message_set_header_values (message, date, from, subject);
 
-   ret = _notmuch_message_index_file (message, message_file);
-   if (ret)
-   goto DONE;
-   } else {
+   ret = _notmuch_message_index_file (message, message_file);
+   if (ret)
+   goto DONE;
+
+   if (! is_new && !is_ghost)
ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-   }
 
_notmuch_message_sync (message);
 } catch (const Xapian::Error ) {
diff --git a/test/T160-json.sh b/test/T160-json.sh
index ac51895e..07955a2b 100755
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -71,8 +71,8 @@ test_begin_subtest "Format version: too high"
 test_expect_code 21 "notmuch search --format-version=999 \\*"
 
 test_begin_subtest "Show message: multiple filenames"
-add_message "[id]=message...@example.com [filename]=copy1"
-add_message "[id]=message...@example.com [filename]=copy2"
+add_message '[id]=message...@example.com [filename]=copy1 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
+add_message '[id]=message...@example.com [filename]=copy2 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
 cat < EXPECTED
 [
 [
diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh
index ced28a21..f1952555 100755
--- a/test/T670-duplicate-mid.sh
+++ b/test/T670-duplicate-mid.sh
@@ -6,7 +6,6 @@ add_message '[id]="duplicate"' '[subject]="message 1" 
[filename]=copy1'
 add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2'
 
 test_begin_subtest 'Search for second subject'
-test_subtest_known_broken
 cat 

[PATCH 01/10] lib: isolate n_d_add_message and helper functions into own file

[David Bremner]

'database.cc' is becoming a monster, and it's hard to follow what the
various static functions are used for. It turns out that about 1/3 of
this file notmuch_database_add_message and helper functions not used
by any other function. This commit isolates this code into it's own
file.

Some side effects of this refactoring:

- find_doc_ids becomes the non-static (but still private)
  _notmuch_database_find_doc_ids
- a few instances of 'string' have 'std::' prepended, avoiding the
  need for 'using namespace std;' in the new file.
---
 lib/Makefile.local |   1 +
 lib/add-message.cc | 721 
 lib/database-private.h |   8 +
 lib/database.cc| 732 +
 4 files changed, 737 insertions(+), 725 deletions(-)
 create mode 100644 lib/add-message.cc

diff --git a/lib/Makefile.local b/lib/Makefile.local
index d36fd5a0..e29fb081 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -48,6 +48,7 @@ libnotmuch_cxx_srcs = \
$(dir)/directory.cc \
$(dir)/index.cc \
$(dir)/message.cc   \
+   $(dir)/add-message.cc   \
$(dir)/message-property.cc \
$(dir)/query.cc \
$(dir)/query-fp.cc  \
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index ..5fe2c45b
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,721 @@
+#include "database-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } else if (*s == ')') {
+   nesting--;
+   } else if (*s == '\\') {
+   if (*(s+1))
+   s++;
+   }
+   s++;
+   }
+   }
+}
+
+*str = s;
+}
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+static char *
+_parse_message_id (void *ctx, const char *message_id, const char **next)
+{
+const char *s, *end;
+char *result;
+
+if (message_id == NULL || *message_id == '\0')
+   return NULL;
+
+s = message_id;
+
+skip_space_and_comments ();
+
+/* Skip any unstructured text as well. */
+while (*s && *s != '<')
+   s++;
+
+if (*s == '<') {
+   s++;
+} else {
+   if (next)
+   *next = s;
+   return NULL;
+}
+
+skip_space_and_comments ();
+
+end = s;
+while (*end && *end != '>')
+   end++;
+if (next) {
+   if (*end)
+   *next = end + 1;
+   else
+   *next = end;
+}
+
+if (end > s && *end == '>')
+   end--;
+if (end <= s)
+   return NULL;
+
+result = talloc_strndup (ctx, s, end - s + 1);
+
+/* Finally, collapse any whitespace that is within the message-id
+ * itself. */
+{
+   char *r;
+   int len;
+
+   for (r = result, len = strlen (r); *r; r++, len--)
+   if (*r == ' ' || *r == '\t')
+   memmove (r, r+1, len);
+}
+
+return result;
+}
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+ const char *message_id,
+ GHashTable *hash,
+ const char *refs)
+{
+char *ref, *last_ref = NULL;
+
+if (refs == NULL || *refs == '\0')
+   return NULL;
+
+while (*refs) {
+   ref = _parse_message_id (ctx, refs, );
+
+   if (ref && strcmp (ref, message_id)) {
+   g_hash_table_add (hash, ref);
+   last_ref = ref;
+   }
+}
+
+/* The return value of this function is used to add a parent
+ * 

v1.1 index multiple files per message-id, add reindex command

[David Bremner]

This obsoletes

 id:20170414025004.5334-1-da...@tethera.net

But for some reason the key patch 9/10 was missing (at least for me),
so I'm calling this round 1.1.

Most of the comments there apply, except

 - I fixed one memory leak in notmuch_message_reindex
 - I added a test that reindexing didn't mess up properties
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 03/10] lib: factor out message-id parsing to separate file.

[David Bremner]

This is really pure C string parsing, and doesn't need to be mixed in
with the Xapian/C++ layer. Although not strictly necessary, it also
makes it a bit more natural to call _parse_message_id from multiple
compilation units.
---
 lib/Makefile.local|   1 +
 lib/add-message.cc| 106 +-
 lib/message-id.c  |  96 +
 lib/notmuch-private.h |  14 +++
 4 files changed, 112 insertions(+), 105 deletions(-)
 create mode 100644 lib/message-id.c

diff --git a/lib/Makefile.local b/lib/Makefile.local
index e29fb081..643199ad 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -36,6 +36,7 @@ libnotmuch_c_srcs =   \
$(dir)/filenames.c  \
$(dir)/string-list.c\
$(dir)/message-file.c   \
+   $(dir)/message-id.c \
$(dir)/messages.c   \
$(dir)/sha1.c   \
$(dir)/built-with.c \
diff --git a/lib/add-message.cc b/lib/add-message.cc
index 0f09415e..314016a8 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -1,109 +1,5 @@
 #include "database-private.h"
 
-/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
- * a (potentially nested) parenthesized sequence with '\' used to
- * escape any character (including parentheses).
- *
- * If the sequence to be skipped continues to the end of the string,
- * then 'str' will be left pointing at the final terminating '\0'
- * character.
- */
-static void
-skip_space_and_comments (const char **str)
-{
-const char *s;
-
-s = *str;
-while (*s && (isspace (*s) || *s == '(')) {
-   while (*s && isspace (*s))
-   s++;
-   if (*s == '(') {
-   int nesting = 1;
-   s++;
-   while (*s && nesting) {
-   if (*s == '(') {
-   nesting++;
-   } else if (*s == ')') {
-   nesting--;
-   } else if (*s == '\\') {
-   if (*(s+1))
-   s++;
-   }
-   s++;
-   }
-   }
-}
-
-*str = s;
-}
-
-/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
- * comments, and the '<' and '>' delimiters.
- *
- * If not NULL, then *next will be made to point to the first character
- * not parsed, (possibly pointing to the final '\0' terminator.
- *
- * Returns a newly talloc'ed string belonging to 'ctx'.
- *
- * Returns NULL if there is any error parsing the message-id. */
-static char *
-_parse_message_id (void *ctx, const char *message_id, const char **next)
-{
-const char *s, *end;
-char *result;
-
-if (message_id == NULL || *message_id == '\0')
-   return NULL;
-
-s = message_id;
-
-skip_space_and_comments ();
-
-/* Skip any unstructured text as well. */
-while (*s && *s != '<')
-   s++;
-
-if (*s == '<') {
-   s++;
-} else {
-   if (next)
-   *next = s;
-   return NULL;
-}
-
-skip_space_and_comments ();
-
-end = s;
-while (*end && *end != '>')
-   end++;
-if (next) {
-   if (*end)
-   *next = end + 1;
-   else
-   *next = end;
-}
-
-if (end > s && *end == '>')
-   end--;
-if (end <= s)
-   return NULL;
-
-result = talloc_strndup (ctx, s, end - s + 1);
-
-/* Finally, collapse any whitespace that is within the message-id
- * itself. */
-{
-   char *r;
-   int len;
-
-   for (r = result, len = strlen (r); *r; r++, len--)
-   if (*r == ' ' || *r == '\t')
-   memmove (r, r+1, len);
-}
-
-return result;
-}
-
 /* Parse a References header value, putting a (talloc'ed under 'ctx')
  * copy of each referenced message-id into 'hash'.
  *
@@ -126,7 +22,7 @@ parse_references (void *ctx,
return NULL;
 
 while (*refs) {
-   ref = _parse_message_id (ctx, refs, );
+   ref = _notmuch_message_id_parse (ctx, refs, );
 
if (ref && strcmp (ref, message_id)) {
g_hash_table_add (hash, ref);
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index ..d7541d50
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,96 @@
+#include "notmuch-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } 

[PATCH] emacs: tree: bugfix: specify --format-version

[Mark Walters]

Previously notmuch tree did not specify the format-version when
calling notmuch. This meant that when the structured output was
slightly changed (in commit 14c60cf168ac3b0f277188c16e6012b7ebdadde7)
stash filename broke. This fixes this breakage by specifying the
format-version.
---

Bremner pointed out this bug today on irc. In due course we may want
to use format-version=3 and update the helper functions but as none of
notmuch-emacs uses the new format version yet this seem the right fix for now.

Best wishes

Mark



emacs/notmuch-tree.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el
index 7bebdba..d4d4076 100644
--- a/emacs/notmuch-tree.el
+++ b/emacs/notmuch-tree.el
@@ -917,7 +917,7 @@ the same as for the function notmuch-tree."
 (notmuch-tag-clear-cache)
 (let ((proc (notmuch-start-notmuch
 "notmuch-tree" (current-buffer) #'notmuch-tree-process-sentinel
-"show" "--body=false" "--format=sexp"
+"show" "--body=false" "--format=sexp" "--format-version=2"
 message-arg search-args))
  ;; Use a scratch buffer to accumulate partial output.
  ;; This buffer will be killed by the sentinel, which
-- 
2.1.4

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 08/10] lib: add _notmuch_message_remove_indexed_terms

[David Bremner]

Testing will be provided via use in notmuch_message_reindex
---
 lib/message.cc| 54 +++
 lib/notmuch-private.h |  2 ++
 2 files changed, 56 insertions(+)

diff --git a/lib/message.cc b/lib/message.cc
index 5291e846..313660da 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -599,6 +599,59 @@ _notmuch_message_remove_terms (notmuch_message_t *message, 
const char *prefix)
 }
 }
 
+
+/* Remove all terms generated by indexing, i.e. not tags or
+ * properties, along with any automatic tags*/
+notmuch_private_status_t
+_notmuch_message_remove_indexed_terms (notmuch_message_t *message)
+{
+Xapian::TermIterator i;
+
+const std::string
+   id_prefix = _find_prefix ("id"),
+   property_prefix = _find_prefix ("property"),
+   tag_prefix = _find_prefix ("tag"),
+   type_prefix = _find_prefix ("type");
+
+for (i = message->doc.termlist_begin ();
+i != message->doc.termlist_end (); i++) {
+
+   const std::string term = *i;
+
+   if (term.compare (0, type_prefix.size (), type_prefix) == 0)
+   continue;
+
+   if (term.compare (0, id_prefix.size (), id_prefix) == 0)
+   continue;
+
+   if (term.compare (0, property_prefix.size (), property_prefix) == 0)
+   continue;
+
+   if (term.compare (0, tag_prefix.size (), tag_prefix) == 0 &&
+   term.compare (1, strlen("encrypted"), "encrypted") != 0 &&
+   term.compare (1, strlen("signed"), "signed") != 0 &&
+   term.compare (1, strlen("attachment"), "attachment") != 0)
+   continue;
+
+   try {
+   message->doc.remove_term ((*i));
+   message->modified = TRUE;
+   } catch (const Xapian::InvalidArgumentError) {
+   /* Ignore failure to remove non-existent term. */
+   } catch (const Xapian::Error ) {
+   notmuch_database_t *notmuch = message->notmuch;
+
+   if (!notmuch->exception_reported) {
+   _notmuch_database_log(_notmuch_message_database (message), "A 
Xapian exception occurred creating message: %s\n",
+ error.get_msg().c_str());
+   notmuch->exception_reported = TRUE;
+   }
+   return NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+   }
+}
+return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
 /* Return true if p points at "new" or "cur". */
 static bool is_maildir (const char *p)
 {
@@ -646,6 +699,7 @@ _notmuch_message_add_folder_terms (notmuch_message_t 
*message,
 
 talloc_free (folder);
 
+message->modified = TRUE;
 return NOTMUCH_STATUS_SUCCESS;
 }
 
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 69179177..ebfba35d 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -534,6 +534,8 @@ _notmuch_message_add_reply (notmuch_message_t *message,
 notmuch_database_t *
 _notmuch_message_database (notmuch_message_t *message);
 
+void
+_notmuch_message_remove_unprefixed_terms (notmuch_message_t *message);
 /* sha1.c */
 
 char *
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 07/10] WIP: Add message count to summary output

[David Bremner]

This requires a bunch of small changes spread out in different places;
I'd probably break it up if we decide to make this change.
---
 lib/message.cc   |  8 
 lib/notmuch-private.h|  6 ++
 lib/notmuch.h|  6 ++
 lib/string-list.c|  6 ++
 lib/thread.cc|  9 +
 notmuch-search.c | 15 +--
 test/T080-search.sh  |  2 +-
 test/T100-search-by-folder.sh|  4 ++--
 test/T340-maildir-sync.sh|  4 ++--
 test/T370-search-folder-coherence.sh |  2 +-
 test/T500-search-date.sh |  2 +-
 11 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index f8215a49..5291e846 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -946,6 +946,14 @@ notmuch_message_get_filenames (notmuch_message_t *message)
 return _notmuch_filenames_create (message, message->filename_list);
 }
 
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+_notmuch_message_ensure_filename_list (message);
+
+return _notmuch_string_list_length (message->filename_list);
+}
+
 notmuch_bool_t
 notmuch_message_get_flag (notmuch_message_t *message,
  notmuch_message_flag_t flag)
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index f3c058ab..69179177 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -558,6 +558,12 @@ typedef struct visible _notmuch_string_list {
 notmuch_string_list_t *
 _notmuch_string_list_create (const void *ctx);
 
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
 /* Add 'string' to 'list'.
  *
  * The list will create its own talloced copy of 'string'.
diff --git a/lib/notmuch.h b/lib/notmuch.h
index d374dc96..7808434f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1094,6 +1094,9 @@ notmuch_thread_get_thread_id (notmuch_thread_t *thread);
 int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
 /**
  * Get a notmuch_messages_t iterator for the top-level messages in
  * 'thread' in oldest-first order.
@@ -1339,6 +1342,9 @@ notmuch_message_get_thread_id (notmuch_message_t 
*message);
 notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
 /**
  * Get a filename for the email corresponding to 'message'.
  *
diff --git a/lib/string-list.c b/lib/string-list.c
index 43ebe499..9c3ae7ef 100644
--- a/lib/string-list.c
+++ b/lib/string-list.c
@@ -42,6 +42,12 @@ _notmuch_string_list_create (const void *ctx)
 return list;
 }
 
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+return list->length;
+}
+
 void
 _notmuch_string_list_append (notmuch_string_list_t *list,
 const char *string)
diff --git a/lib/thread.cc b/lib/thread.cc
index 561ca5be..d385102b 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -44,6 +44,7 @@ struct visible _notmuch_thread {
 
 GHashTable *message_hash;
 int total_messages;
+int total_files;
 int matched_messages;
 time_t oldest;
 time_t newest;
@@ -266,6 +267,7 @@ _thread_add_message (notmuch_thread_t *thread,
 _notmuch_message_list_add_message (thread->message_list,
   talloc_steal (thread, message));
 thread->total_messages++;
+thread->total_files += notmuch_message_count_files (message);
 
 g_hash_table_insert (thread->message_hash,
 xstrdup (notmuch_message_get_message_id (message)),
@@ -495,6 +497,7 @@ _notmuch_thread_create (void *ctx,
  free, NULL);
 
 thread->total_messages = 0;
+thread->total_files = 0;
 thread->matched_messages = 0;
 thread->oldest = 0;
 thread->newest = 0;
@@ -567,6 +570,12 @@ notmuch_thread_get_total_messages (notmuch_thread_t 
*thread)
 }
 
 int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+return thread->total_files;
+}
+
+int
 notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
 {
 return thread->matched_messages;
diff --git a/notmuch-search.c b/notmuch-search.c
index 019e14ee..380e9d8f 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -160,6 +160,7 @@ do_search_threads (search_context_t *ctx)
const char *subject = notmuch_thread_get_subject (thread);
const char *thread_id = notmuch_thread_get_thread_id (thread);
int matched = notmuch_thread_get_matched_messages (thread);
+   int files = notmuch_thread_get_total_files (thread);
int total = notmuch_thread_get_total_messages (thread);
const char *relative_date = NULL;
notmuch_bool_t first_tag = TRUE;
@@ -175,13 +176,23 @@ do_search_threads 

[PATCH 02/10] lib/n_d_add_message: refactor test for new/ghost messages

[David Bremner]

The switch is easier to understand than the side effects in the if
test. It also potentially allows us more flexibility in breaking up
this function into smaller pieces, since passing private_status around
is icky.
---
 lib/add-message.cc | 23 +--
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 5fe2c45b..0f09415e 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -570,7 +570,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 notmuch_message_t *message = NULL;
 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
 notmuch_private_status_t private_status;
-notmuch_bool_t is_ghost = false;
+notmuch_bool_t is_ghost = FALSE, is_new = FALSE;
 
 const char *date, *header;
 const char *from, *to, *subject;
@@ -655,7 +655,17 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
talloc_free (message_id);
 
-   if (message == NULL) {
+   /* We cannot call notmuch_message_get_flag for a new message */
+   switch (private_status) {
+   case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+   is_ghost = FALSE;
+   is_new = TRUE;
+   break;
+   case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+   is_ghost = notmuch_message_get_flag (message, 
NOTMUCH_MESSAGE_FLAG_GHOST);
+   is_new = FALSE;
+   break;
+   default:
ret = COERCE_STATUS (private_status,
 "Unexpected status value from 
_notmuch_message_create_for_message_id");
goto DONE;
@@ -663,18 +673,11 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
_notmuch_message_add_filename (message, filename);
 
-   /* Is this a newly created message object or a ghost
-* message?  We have to be slightly careful: if this is a
-* blank message, it's not safe to call
-* notmuch_message_get_flag yet. */
-   if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND ||
-   (is_ghost = notmuch_message_get_flag (
-   message, NOTMUCH_MESSAGE_FLAG_GHOST))) {
+   if (is_new || is_ghost) {
_notmuch_message_add_term (message, "type", "mail");
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-
ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
if (ret)
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 09/10] lib: add notmuch_message_reindex

[David Bremner]

From: Daniel Kahn Gillmor 

This new function asks the database to reindex a given message.
The parameter `indexopts` is currently ignored, but is intended to
provide an extensible API to support e.g. changing the encryption or
filtering status (e.g. whether and how certain non-plaintext parts are
indexed).
---
 lib/add-message.cc|   2 +-
 lib/message.cc| 108 +-
 lib/notmuch-private.h |   6 +++
 lib/notmuch.h |  15 +++
 4 files changed, 129 insertions(+), 2 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index ae9b14a7..26405742 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -220,7 +220,7 @@ _my_talloc_free_for_g_hash (void *ptr)
 talloc_free (ptr);
 }
 
-static notmuch_status_t
+notmuch_status_t
 _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
   notmuch_message_t *message,
   notmuch_message_file_t *message_file,
diff --git a/lib/message.cc b/lib/message.cc
index 313660da..1cc59dfa 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -579,7 +579,9 @@ void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
 Xapian::TermIterator i;
-size_t prefix_len = strlen (prefix);
+size_t prefix_len = 0;
+
+prefix_len = strlen (prefix);
 
 while (1) {
i = message->doc.termlist_begin ();
@@ -1934,3 +1936,107 @@ _notmuch_message_frozen (notmuch_message_t *message)
 {
 return message->frozen;
 }
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+notmuch_param_t unused (*indexopts))
+{
+notmuch_database_t *notmuch = NULL;
+notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+notmuch_private_status_t private_status;
+notmuch_filenames_t *orig_filenames = NULL;
+const char *orig_thread_id = NULL;
+notmuch_message_file_t *message_file = NULL;
+
+int found = 0;
+
+if (message == NULL)
+   return NOTMUCH_STATUS_NULL_POINTER;
+
+/* Save in case we need to delete message */
+orig_thread_id = notmuch_message_get_thread_id (message);
+if (!orig_thread_id) {
+   /* XXX TODO: make up new error return? */
+   INTERNAL_ERROR ("message without thread-id");
+}
+
+/* strdup it because the metadata may be invalidated */
+orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+notmuch = _notmuch_message_database (message);
+
+ret = _notmuch_database_ensure_writable (notmuch);
+if (ret)
+   return ret;
+
+orig_filenames = notmuch_message_get_filenames (message);
+
+private_status = _notmuch_message_remove_indexed_terms (message);
+if (private_status) {
+   ret = COERCE_STATUS(private_status, "error removing terms");
+   goto DONE;
+}
+
+/* re-add the filenames with the associated indexopts */
+for (; notmuch_filenames_valid (orig_filenames);
+notmuch_filenames_move_to_next (orig_filenames)) {
+
+   const char *date;
+   const char *from, *to, *subject;
+   char *message_id = NULL;
+   const char *thread_id = NULL;
+
+   const char *filename = notmuch_filenames_get (orig_filenames);
+
+   message_file = _notmuch_message_file_open (notmuch, filename);
+   if (message_file == NULL)
+   continue;
+
+   ret = _notmuch_message_file_get_headers (message_file,
+, , , ,
+_id);
+   if (ret)
+   goto DONE;
+
+   /* XXX TODO: deal with changing message id? */
+
+   _notmuch_message_add_filename (message, filename);
+
+   ret = _notmuch_database_link_message_to_parents (notmuch, message,
+message_file,
+_id);
+   if (ret)
+   goto DONE;
+
+   if (thread_id == NULL)
+   thread_id = orig_thread_id;
+
+   _notmuch_message_add_term (message, "thread", thread_id);
+   _notmuch_message_set_header_values (message, date, from, subject);
+
+   ret = _notmuch_message_index_file (message, message_file);
+
+   if (ret == NOTMUCH_STATUS_FILE_ERROR)
+   continue;
+   if (ret)
+   goto DONE;
+
+   found++;
+}
+if (found == 0) {
+   /* put back thread id to help cleanup */
+   _notmuch_message_add_term (message, "thread", orig_thread_id);
+   ret = _notmuch_message_delete (message);
+   if (ret)
+   return ret;
+} else {
+   _notmuch_message_sync (message);
+}
+
+ DONE:
+if (message_file)
+   _notmuch_message_file_close (message_file);
+
+/* XXX TODO destroy orig_filenames? */
+return ret;
+}
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index ebfba35d..beb50033 100644
--- a/lib/notmuch-private.h
+++ 

[PATCH 04/10] lib: refactor notmuch_database_add_message header parsing

[David Bremner]

This function is large and hard to understand and modify. Start to
break it down into meaningful pieces.
---
 lib/add-message.cc| 54 +++-
 lib/message-file.c| 86 +++
 lib/notmuch-private.h | 11 +++
 3 files changed, 101 insertions(+), 50 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 314016a8..2922eaa9 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -468,7 +468,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 notmuch_private_status_t private_status;
 notmuch_bool_t is_ghost = FALSE, is_new = FALSE;
 
-const char *date, *header;
+const char *date;
 const char *from, *to, *subject;
 char *message_id = NULL;
 
@@ -489,57 +489,12 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 if (ret)
goto DONE;
 
-/* Parse message up front to get better error status. */
-ret = _notmuch_message_file_parse (message_file);
+ret = _notmuch_message_file_get_headers (message_file,
+, , , ,
+_id);
 if (ret)
goto DONE;
 
-/* Before we do any real work, (especially before doing a
- * potential SHA-1 computation on the entire file's contents),
- * let's make sure that what we're looking at looks like an
- * actual email message.
- */
-from = _notmuch_message_file_get_header (message_file, "from");
-subject = _notmuch_message_file_get_header (message_file, "subject");
-to = _notmuch_message_file_get_header (message_file, "to");
-
-if ((from == NULL || *from == '\0') &&
-   (subject == NULL || *subject == '\0') &&
-   (to == NULL || *to == '\0')) {
-   ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
-   goto DONE;
-}
-
-/* Now that we're sure it's mail, the first order of business
- * is to find a message ID (or else create one ourselves).
- */
-header = _notmuch_message_file_get_header (message_file, "message-id");
-if (header && *header != '\0') {
-   message_id = _parse_message_id (message_file, header, NULL);
-
-   /* So the header value isn't RFC-compliant, but it's
-* better than no message-id at all.
-*/
-   if (message_id == NULL)
-   message_id = talloc_strdup (message_file, header);
-}
-
-if (message_id == NULL ) {
-   /* No message-id at all, let's generate one by taking a
-* hash over the file's contents.
-*/
-   char *sha1 = _notmuch_sha1_of_file (filename);
-
-   /* If that failed too, something is really wrong. Give up. */
-   if (sha1 == NULL) {
-   ret = NOTMUCH_STATUS_FILE_ERROR;
-   goto DONE;
-   }
-
-   message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
-   free (sha1);
-}
-
 try {
/* Now that we have a message ID, we get a message object,
 * (which may or may not reference an existing document in the
@@ -579,7 +534,6 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (ret)
goto DONE;
 
-   date = _notmuch_message_file_get_header (message_file, "date");
_notmuch_message_set_header_values (message, date, from, subject);
 
ret = _notmuch_message_index_file (message, message_file);
diff --git a/lib/message-file.c b/lib/message-file.c
index db18b163..70526ef0 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -92,6 +92,12 @@ _notmuch_message_file_open (notmuch_database_t *notmuch,
 return _notmuch_message_file_open_ctx (notmuch, NULL, filename);
 }
 
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message_file)
+{
+return message_file->filename;
+}
+
 void
 _notmuch_message_file_close (notmuch_message_file_t *message)
 {
@@ -304,3 +310,83 @@ _notmuch_message_file_get_header (notmuch_message_file_t 
*message,
 
 return decoded;
 }
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+  const char **from_out,
+  const char **subject_out,
+  const char **to_out,
+  const char **date_out,
+  char **message_id_out)
+{
+notmuch_status_t ret;
+const char *header;
+const char *from, *to, *subject, *date;
+char *message_id = NULL;
+
+/* Parse message up front to get better error status. */
+ret = _notmuch_message_file_parse (message_file);
+if (ret)
+   goto DONE;
+
+/* Before we do any real work, (especially before doing a
+ * potential SHA-1 computation on the entire file's contents),
+ * let's make sure that what we're looking at looks like an
+ * actual email message.
+ */
+from = _notmuch_message_file_get_header (message_file, "from");
+

[PATCH 05/10] test: add known broken tests for duplicate message id

[David Bremner]

There are many other problems that could be tested, but these ones we
have some hope of fixing because it doesn't require UI changes, just
indexing changes.
---
 test/T670-duplicate-mid.sh | 28 
 1 file changed, 28 insertions(+)
 create mode 100755 test/T670-duplicate-mid.sh

diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh
new file mode 100755
index ..ced28a21
--- /dev/null
+++ b/test/T670-duplicate-mid.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+test_description="duplicate message ids"
+. ./test-lib.sh || exit 1
+
+add_message '[id]="duplicate"' '[subject]="message 1" [filename]=copy1'
+add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2'
+
+test_begin_subtest 'Search for second subject'
+test_subtest_known_broken
+cat  
OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+add_message '[id]="duplicate"' '[body]="sekrit" [filename]=copy3'
+test_begin_subtest 'search for body in duplicate file'
+test_subtest_known_broken
+cat  OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 10/10] add "notmuch reindex" subcommand

[David Bremner]

From: Daniel Kahn Gillmor 

This new subcommand takes a set of search terms, and re-indexes the
list of matching messages.
---
 Makefile.local|   1 +
 doc/conf.py   |   4 ++
 doc/index.rst |   1 +
 doc/man1/notmuch-reindex.rst  |  29 +
 doc/man1/notmuch.rst  |   4 +-
 doc/man7/notmuch-search-terms.rst |   7 +-
 notmuch-client.h  |   3 +
 notmuch-reindex.c | 134 ++
 notmuch.c |   2 +
 performance-test/M04-reindex.sh   |  11 
 performance-test/T03-reindex.sh   |  13 
 test/T670-duplicate-mid.sh|   7 ++
 test/T700-reindex.sh  |  79 ++
 13 files changed, 291 insertions(+), 4 deletions(-)
 create mode 100644 doc/man1/notmuch-reindex.rst
 create mode 100644 notmuch-reindex.c
 create mode 100755 performance-test/M04-reindex.sh
 create mode 100755 performance-test/T03-reindex.sh
 create mode 100755 test/T700-reindex.sh

diff --git a/Makefile.local b/Makefile.local
index 03eafaaa..c6e272bc 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -222,6 +222,7 @@ notmuch_client_srcs =   \
notmuch-dump.c  \
notmuch-insert.c\
notmuch-new.c   \
+   notmuch-reindex.c   \
notmuch-reply.c \
notmuch-restore.c   \
notmuch-search.c\
diff --git a/doc/conf.py b/doc/conf.py
index a3d82696..aa864b3c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -95,6 +95,10 @@ man_pages = [
  u'incorporate new mail into the notmuch database',
  [notmuch_authors], 1),
 
+('man1/notmuch-reindex', 'notmuch-reindex',
+ u're-index matching messages',
+ [notmuch_authors], 1),
+
 ('man1/notmuch-reply', 'notmuch-reply',
  u'constructs a reply template for a set of messages',
  [notmuch_authors], 1),
diff --git a/doc/index.rst b/doc/index.rst
index 344606d9..aa6c9f40 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -18,6 +18,7 @@ Contents:
man5/notmuch-hooks
man1/notmuch-insert
man1/notmuch-new
+   man1/notmuch-reindex
man1/notmuch-reply
man1/notmuch-restore
man1/notmuch-search
diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst
new file mode 100644
index ..e39cc4ee
--- /dev/null
+++ b/doc/man1/notmuch-reindex.rst
@@ -0,0 +1,29 @@
+===
+notmuch-reindex
+===
+
+SYNOPSIS
+
+
+**notmuch** **reindex** [*option* ...] <*search-term*> ...
+
+DESCRIPTION
+===
+
+Re-index all messages matching the search terms.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<*search-term*\ >.
+
+The **reindex** command searches for all messages matching the
+supplied search terms, and re-creates the full-text index on these
+messages using the supplied options.
+
+SEE ALSO
+
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch.rst b/doc/man1/notmuch.rst
index fbd7f381..b2a8376e 100644
--- a/doc/man1/notmuch.rst
+++ b/doc/man1/notmuch.rst
@@ -149,8 +149,8 @@ SEE ALSO
 
 **notmuch-address(1)**, **notmuch-compact(1)**, **notmuch-config(1)**,
 **notmuch-count(1)**, **notmuch-dump(1)**, **notmuch-hooks(5)**,
-**notmuch-insert(1)**, **notmuch-new(1)**, **notmuch-reply(1)**,
-**notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-insert(1)**, **notmuch-new(1)**, **notmuch-reindex(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
 **notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
 
 The notmuch website: **https://notmuchmail.org**
diff --git a/doc/man7/notmuch-search-terms.rst 
b/doc/man7/notmuch-search-terms.rst
index 47cab48d..dd76972e 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -9,6 +9,8 @@ SYNOPSIS
 
 **notmuch** **dump** [--format=(batch-tag|sup)] [--] [--output=<*file*>] [--] 
[<*search-term*> ...]
 
+**notmuch** **reindex** [option ...] <*search-term*> ...
+
 **notmuch** **search** [option ...] <*search-term*> ...
 
 **notmuch** **show** [option ...] <*search-term*> ...
@@ -421,5 +423,6 @@ SEE ALSO
 
 **notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
 **notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
-**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
-**notmuch-search(1)**, **notmuch-show(1)**, **notmuch-tag(1)**
+**notmuch-new(1)**, **notmuch-reindex(1)**, **notmuch-reply(1)**,
+**notmuch-restore(1)**, **notmuch-search(1)**, **notmuch-show(1)**,
+**notmuch-tag(1)**
diff --git a/notmuch-client.h b/notmuch-client.h
index a6f70eae..ab7138c6 100644
--- a/notmuch-client.h
+++ 

[PATCH 07/10] WIP: Add message count to summary output

[David Bremner]

This requires a bunch of small changes spread out in different places;
I'd probably break it up if we decide to make this change.
---
 lib/message.cc   |  8 
 lib/notmuch-private.h|  6 ++
 lib/notmuch.h|  6 ++
 lib/string-list.c|  6 ++
 lib/thread.cc|  9 +
 notmuch-search.c | 15 +--
 test/T080-search.sh  |  2 +-
 test/T100-search-by-folder.sh|  4 ++--
 test/T340-maildir-sync.sh|  4 ++--
 test/T370-search-folder-coherence.sh |  2 +-
 test/T500-search-date.sh |  2 +-
 11 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/lib/message.cc b/lib/message.cc
index f8215a49..5291e846 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -946,6 +946,14 @@ notmuch_message_get_filenames (notmuch_message_t *message)
 return _notmuch_filenames_create (message, message->filename_list);
 }
 
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+_notmuch_message_ensure_filename_list (message);
+
+return _notmuch_string_list_length (message->filename_list);
+}
+
 notmuch_bool_t
 notmuch_message_get_flag (notmuch_message_t *message,
  notmuch_message_flag_t flag)
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index f3c058ab..69179177 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -558,6 +558,12 @@ typedef struct visible _notmuch_string_list {
 notmuch_string_list_t *
 _notmuch_string_list_create (const void *ctx);
 
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
 /* Add 'string' to 'list'.
  *
  * The list will create its own talloced copy of 'string'.
diff --git a/lib/notmuch.h b/lib/notmuch.h
index d374dc96..7808434f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1094,6 +1094,9 @@ notmuch_thread_get_thread_id (notmuch_thread_t *thread);
 int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
 /**
  * Get a notmuch_messages_t iterator for the top-level messages in
  * 'thread' in oldest-first order.
@@ -1339,6 +1342,9 @@ notmuch_message_get_thread_id (notmuch_message_t 
*message);
 notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
 /**
  * Get a filename for the email corresponding to 'message'.
  *
diff --git a/lib/string-list.c b/lib/string-list.c
index 43ebe499..9c3ae7ef 100644
--- a/lib/string-list.c
+++ b/lib/string-list.c
@@ -42,6 +42,12 @@ _notmuch_string_list_create (const void *ctx)
 return list;
 }
 
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+return list->length;
+}
+
 void
 _notmuch_string_list_append (notmuch_string_list_t *list,
 const char *string)
diff --git a/lib/thread.cc b/lib/thread.cc
index 561ca5be..d385102b 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -44,6 +44,7 @@ struct visible _notmuch_thread {
 
 GHashTable *message_hash;
 int total_messages;
+int total_files;
 int matched_messages;
 time_t oldest;
 time_t newest;
@@ -266,6 +267,7 @@ _thread_add_message (notmuch_thread_t *thread,
 _notmuch_message_list_add_message (thread->message_list,
   talloc_steal (thread, message));
 thread->total_messages++;
+thread->total_files += notmuch_message_count_files (message);
 
 g_hash_table_insert (thread->message_hash,
 xstrdup (notmuch_message_get_message_id (message)),
@@ -495,6 +497,7 @@ _notmuch_thread_create (void *ctx,
  free, NULL);
 
 thread->total_messages = 0;
+thread->total_files = 0;
 thread->matched_messages = 0;
 thread->oldest = 0;
 thread->newest = 0;
@@ -567,6 +570,12 @@ notmuch_thread_get_total_messages (notmuch_thread_t 
*thread)
 }
 
 int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+return thread->total_files;
+}
+
+int
 notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
 {
 return thread->matched_messages;
diff --git a/notmuch-search.c b/notmuch-search.c
index 019e14ee..380e9d8f 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -160,6 +160,7 @@ do_search_threads (search_context_t *ctx)
const char *subject = notmuch_thread_get_subject (thread);
const char *thread_id = notmuch_thread_get_thread_id (thread);
int matched = notmuch_thread_get_matched_messages (thread);
+   int files = notmuch_thread_get_total_files (thread);
int total = notmuch_thread_get_total_messages (thread);
const char *relative_date = NULL;
notmuch_bool_t first_tag = TRUE;
@@ -175,13 +176,23 @@ do_search_threads 

[PATCH 09/10] lib: add notmuch_message_reindex

[David Bremner]

From: Daniel Kahn Gillmor 

This new function asks the database to reindex a given message.
The parameter `indexopts` is currently ignored, but is intended to
provide an extensible API to support e.g. changing the encryption or
filtering status (e.g. whether and how certain non-plaintext parts are
indexed).
---
 lib/add-message.cc|   2 +-
 lib/message.cc| 108 +-
 lib/notmuch-private.h |   6 +++
 lib/notmuch.h |  15 +++
 4 files changed, 129 insertions(+), 2 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index ae9b14a7..26405742 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -220,7 +220,7 @@ _my_talloc_free_for_g_hash (void *ptr)
 talloc_free (ptr);
 }
 
-static notmuch_status_t
+notmuch_status_t
 _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
   notmuch_message_t *message,
   notmuch_message_file_t *message_file,
diff --git a/lib/message.cc b/lib/message.cc
index 313660da..6613793e 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -579,7 +579,9 @@ void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
 Xapian::TermIterator i;
-size_t prefix_len = strlen (prefix);
+size_t prefix_len = 0;
+
+prefix_len = strlen (prefix);
 
 while (1) {
i = message->doc.termlist_begin ();
@@ -1934,3 +1936,107 @@ _notmuch_message_frozen (notmuch_message_t *message)
 {
 return message->frozen;
 }
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+notmuch_param_t unused (*indexopts))
+{
+notmuch_database_t *notmuch = NULL;
+notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+notmuch_private_status_t private_status;
+notmuch_filenames_t *orig_filenames = NULL;
+const char *orig_thread_id = NULL;
+notmuch_message_file_t *message_file = NULL;
+
+int found = 0;
+
+if (message == NULL)
+   return NOTMUCH_STATUS_NULL_POINTER;
+
+/* Save in case we need to delete message */
+orig_thread_id = notmuch_message_get_thread_id (message);
+if (!orig_thread_id) {
+   /* XXX TODO: make up new error return? */
+   INTERNAL_ERROR ("message without thread-id");
+}
+
+/* strdup it because the metadata may be invalidated */
+orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+notmuch = _notmuch_message_database (message);
+
+ret = _notmuch_database_ensure_writable (notmuch);
+if (ret)
+   return ret;
+
+orig_filenames = notmuch_message_get_filenames (message);
+
+private_status = _notmuch_message_remove_indexed_terms (message);
+if (private_status) {
+   ret = COERCE_STATUS(private_status, "error removing terms");
+   goto DONE;
+}
+
+/* re-add the filenames with the associated indexopts */
+for (; notmuch_filenames_valid (orig_filenames);
+notmuch_filenames_move_to_next (orig_filenames)) {
+
+   const char *date;
+   const char *from, *to, *subject;
+   char *message_id = NULL;
+   const char *thread_id = NULL;
+
+   const char *filename = notmuch_filenames_get (orig_filenames);
+
+   message_file = _notmuch_message_file_open (notmuch, filename);
+   if (message_file == NULL)
+   continue;
+
+   ret = _notmuch_message_file_get_headers (message_file,
+, , , ,
+_id);
+   if (ret)
+   goto DONE;
+
+   /* XXX TODO: deal with changing message id? */
+
+   _notmuch_message_add_filename (message, filename);
+
+   ret = _notmuch_database_link_message_to_parents (notmuch, message,
+message_file,
+_id);
+   if (ret)
+   goto DONE;
+
+   if (thread_id == NULL)
+   thread_id = orig_thread_id;
+
+   _notmuch_message_add_term (message, "thread", thread_id);
+   _notmuch_message_set_header_values (message, date, from, subject);
+
+   ret = _notmuch_message_index_file (message, message_file);
+
+   if (ret == NOTMUCH_STATUS_FILE_ERROR)
+   continue;
+   if (ret)
+   goto DONE;
+
+   found++;
+   _notmuch_message_file_close (message_file);
+   message_file = NULL;
+}
+if (found == 0) {
+   /* put back thread id to help cleanup */
+   _notmuch_message_add_term (message, "thread", orig_thread_id);
+   ret = _notmuch_message_delete (message);
+} else {
+   _notmuch_message_sync (message);
+}
+
+ DONE:
+if (message_file)
+   _notmuch_message_file_close (message_file);
+
+/* XXX TODO destroy orig_filenames? */
+return ret;
+}
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index ebfba35d..beb50033 

[PATCH 06/12] lib: index message files with duplicate message-ids

[David Bremner]

The corresponding xapian document just gets more terms added to it,
but this doesn't seem to break anything. Values on the other hand get
overwritten, which is a bit annoying, but arguably it is not worse to
take the values (from, subject, date) from the last file indexed
rather than the first.
---
 lib/add-message.cc | 20 +++-
 test/T160-json.sh  |  4 ++--
 test/T670-duplicate-mid.sh |  2 --
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 2922eaa9..ae9b14a7 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -529,19 +529,21 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-   ret = _notmuch_database_link_message (notmuch, message,
+   }
+
+   ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
-   if (ret)
-   goto DONE;
+   if (ret)
+   goto DONE;
 
-   _notmuch_message_set_header_values (message, date, from, subject);
+   _notmuch_message_set_header_values (message, date, from, subject);
 
-   ret = _notmuch_message_index_file (message, message_file);
-   if (ret)
-   goto DONE;
-   } else {
+   ret = _notmuch_message_index_file (message, message_file);
+   if (ret)
+   goto DONE;
+
+   if (! is_new && !is_ghost)
ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-   }
 
_notmuch_message_sync (message);
 } catch (const Xapian::Error ) {
diff --git a/test/T160-json.sh b/test/T160-json.sh
index ac51895e..07955a2b 100755
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -71,8 +71,8 @@ test_begin_subtest "Format version: too high"
 test_expect_code 21 "notmuch search --format-version=999 \\*"
 
 test_begin_subtest "Show message: multiple filenames"
-add_message "[id]=message...@example.com [filename]=copy1"
-add_message "[id]=message...@example.com [filename]=copy2"
+add_message '[id]=message...@example.com [filename]=copy1 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
+add_message '[id]=message...@example.com [filename]=copy2 [date]="Fri, 05 Jan 
2001 15:43:52 +"'
 cat < EXPECTED
 [
 [
diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh
index ced28a21..f1952555 100755
--- a/test/T670-duplicate-mid.sh
+++ b/test/T670-duplicate-mid.sh
@@ -6,7 +6,6 @@ add_message '[id]="duplicate"' '[subject]="message 1" 
[filename]=copy1'
 add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2'
 
 test_begin_subtest 'Search for second subject'
-test_subtest_known_broken
 cat 

[PATCH 01/12] lib: isolate n_d_add_message and helper functions into own file

[David Bremner]

'database.cc' is becoming a monster, and it's hard to follow what the
various static functions are used for. It turns out that about 1/3 of
this file notmuch_database_add_message and helper functions not used
by any other function. This commit isolates this code into it's own
file.

Some side effects of this refactoring:

- find_doc_ids becomes the non-static (but still private)
  _notmuch_database_find_doc_ids
- a few instances of 'string' have 'std::' prepended, avoiding the
  need for 'using namespace std;' in the new file.
---
 lib/Makefile.local |   1 +
 lib/add-message.cc | 721 
 lib/database-private.h |   8 +
 lib/database.cc| 732 +
 4 files changed, 737 insertions(+), 725 deletions(-)
 create mode 100644 lib/add-message.cc

diff --git a/lib/Makefile.local b/lib/Makefile.local
index d36fd5a0..e29fb081 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -48,6 +48,7 @@ libnotmuch_cxx_srcs = \
$(dir)/directory.cc \
$(dir)/index.cc \
$(dir)/message.cc   \
+   $(dir)/add-message.cc   \
$(dir)/message-property.cc \
$(dir)/query.cc \
$(dir)/query-fp.cc  \
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index ..5fe2c45b
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,721 @@
+#include "database-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } else if (*s == ')') {
+   nesting--;
+   } else if (*s == '\\') {
+   if (*(s+1))
+   s++;
+   }
+   s++;
+   }
+   }
+}
+
+*str = s;
+}
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+static char *
+_parse_message_id (void *ctx, const char *message_id, const char **next)
+{
+const char *s, *end;
+char *result;
+
+if (message_id == NULL || *message_id == '\0')
+   return NULL;
+
+s = message_id;
+
+skip_space_and_comments ();
+
+/* Skip any unstructured text as well. */
+while (*s && *s != '<')
+   s++;
+
+if (*s == '<') {
+   s++;
+} else {
+   if (next)
+   *next = s;
+   return NULL;
+}
+
+skip_space_and_comments ();
+
+end = s;
+while (*end && *end != '>')
+   end++;
+if (next) {
+   if (*end)
+   *next = end + 1;
+   else
+   *next = end;
+}
+
+if (end > s && *end == '>')
+   end--;
+if (end <= s)
+   return NULL;
+
+result = talloc_strndup (ctx, s, end - s + 1);
+
+/* Finally, collapse any whitespace that is within the message-id
+ * itself. */
+{
+   char *r;
+   int len;
+
+   for (r = result, len = strlen (r); *r; r++, len--)
+   if (*r == ' ' || *r == '\t')
+   memmove (r, r+1, len);
+}
+
+return result;
+}
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+ const char *message_id,
+ GHashTable *hash,
+ const char *refs)
+{
+char *ref, *last_ref = NULL;
+
+if (refs == NULL || *refs == '\0')
+   return NULL;
+
+while (*refs) {
+   ref = _parse_message_id (ctx, refs, );
+
+   if (ref && strcmp (ref, message_id)) {
+   g_hash_table_add (hash, ref);
+   last_ref = ref;
+   }
+}
+
+/* The return value of this function is used to add a parent
+ * 

Re: [notmuch] Re: v3 of regexp search for mid/folder/path

[Tomi Ollila]

On Sun, May 07 2017, David Bremner wrote:

> David Bremner  writes:
>
>> No sooner posted than I realized it had a bug: the previous version
>> compared against the prefixed term so anchored searches failed.
>>
>> I've also included some tests for the new features in this version.
>>
>> Below is an interdiff against v1
>
> Gauteh reported success with these patches on IRC. Anyone want more time
> to review?

Nope, but fix s/implimentation/implementation/ in 2/2 commit message :D

Tomi

>
> d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 03/12] lib: factor out message-id parsing to separate file.

[David Bremner]

This is really pure C string parsing, and doesn't need to be mixed in
with the Xapian/C++ layer. Although not strictly necessary, it also
makes it a bit more natural to call _parse_message_id from multiple
compilation units.
---
 lib/Makefile.local|   1 +
 lib/add-message.cc| 106 +-
 lib/message-id.c  |  96 +
 lib/notmuch-private.h |  14 +++
 4 files changed, 112 insertions(+), 105 deletions(-)
 create mode 100644 lib/message-id.c

diff --git a/lib/Makefile.local b/lib/Makefile.local
index e29fb081..643199ad 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -36,6 +36,7 @@ libnotmuch_c_srcs =   \
$(dir)/filenames.c  \
$(dir)/string-list.c\
$(dir)/message-file.c   \
+   $(dir)/message-id.c \
$(dir)/messages.c   \
$(dir)/sha1.c   \
$(dir)/built-with.c \
diff --git a/lib/add-message.cc b/lib/add-message.cc
index 0f09415e..314016a8 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -1,109 +1,5 @@
 #include "database-private.h"
 
-/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
- * a (potentially nested) parenthesized sequence with '\' used to
- * escape any character (including parentheses).
- *
- * If the sequence to be skipped continues to the end of the string,
- * then 'str' will be left pointing at the final terminating '\0'
- * character.
- */
-static void
-skip_space_and_comments (const char **str)
-{
-const char *s;
-
-s = *str;
-while (*s && (isspace (*s) || *s == '(')) {
-   while (*s && isspace (*s))
-   s++;
-   if (*s == '(') {
-   int nesting = 1;
-   s++;
-   while (*s && nesting) {
-   if (*s == '(') {
-   nesting++;
-   } else if (*s == ')') {
-   nesting--;
-   } else if (*s == '\\') {
-   if (*(s+1))
-   s++;
-   }
-   s++;
-   }
-   }
-}
-
-*str = s;
-}
-
-/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
- * comments, and the '<' and '>' delimiters.
- *
- * If not NULL, then *next will be made to point to the first character
- * not parsed, (possibly pointing to the final '\0' terminator.
- *
- * Returns a newly talloc'ed string belonging to 'ctx'.
- *
- * Returns NULL if there is any error parsing the message-id. */
-static char *
-_parse_message_id (void *ctx, const char *message_id, const char **next)
-{
-const char *s, *end;
-char *result;
-
-if (message_id == NULL || *message_id == '\0')
-   return NULL;
-
-s = message_id;
-
-skip_space_and_comments ();
-
-/* Skip any unstructured text as well. */
-while (*s && *s != '<')
-   s++;
-
-if (*s == '<') {
-   s++;
-} else {
-   if (next)
-   *next = s;
-   return NULL;
-}
-
-skip_space_and_comments ();
-
-end = s;
-while (*end && *end != '>')
-   end++;
-if (next) {
-   if (*end)
-   *next = end + 1;
-   else
-   *next = end;
-}
-
-if (end > s && *end == '>')
-   end--;
-if (end <= s)
-   return NULL;
-
-result = talloc_strndup (ctx, s, end - s + 1);
-
-/* Finally, collapse any whitespace that is within the message-id
- * itself. */
-{
-   char *r;
-   int len;
-
-   for (r = result, len = strlen (r); *r; r++, len--)
-   if (*r == ' ' || *r == '\t')
-   memmove (r, r+1, len);
-}
-
-return result;
-}
-
 /* Parse a References header value, putting a (talloc'ed under 'ctx')
  * copy of each referenced message-id into 'hash'.
  *
@@ -126,7 +22,7 @@ parse_references (void *ctx,
return NULL;
 
 while (*refs) {
-   ref = _parse_message_id (ctx, refs, );
+   ref = _notmuch_message_id_parse (ctx, refs, );
 
if (ref && strcmp (ref, message_id)) {
g_hash_table_add (hash, ref);
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index ..d7541d50
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,96 @@
+#include "notmuch-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+const char *s;
+
+s = *str;
+while (*s && (isspace (*s) || *s == '(')) {
+   while (*s && isspace (*s))
+   s++;
+   if (*s == '(') {
+   int nesting = 1;
+   s++;
+   while (*s && nesting) {
+   if (*s == '(') {
+   nesting++;
+   } 

[PATCH 12/12] add "notmuch reindex" subcommand

[David Bremner]

From: Daniel Kahn Gillmor 

This new subcommand takes a set of search terms, and re-indexes the
list of matching messages.
---
 Makefile.local|   1 +
 doc/conf.py   |   4 ++
 doc/index.rst |   1 +
 doc/man1/notmuch-reindex.rst  |  29 +
 doc/man1/notmuch.rst  |   4 +-
 doc/man7/notmuch-search-terms.rst |   7 +-
 notmuch-client.h  |   3 +
 notmuch-reindex.c | 134 ++
 notmuch.c |   2 +
 performance-test/M04-reindex.sh   |  11 
 performance-test/T03-reindex.sh   |  13 
 test/T670-duplicate-mid.sh|   7 ++
 test/T700-reindex.sh  |  79 ++
 13 files changed, 291 insertions(+), 4 deletions(-)
 create mode 100644 doc/man1/notmuch-reindex.rst
 create mode 100644 notmuch-reindex.c
 create mode 100755 performance-test/M04-reindex.sh
 create mode 100755 performance-test/T03-reindex.sh
 create mode 100755 test/T700-reindex.sh

diff --git a/Makefile.local b/Makefile.local
index 3d3474e0..b4348fe3 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -224,6 +224,7 @@ notmuch_client_srcs =   \
notmuch-dump.c  \
notmuch-insert.c\
notmuch-new.c   \
+   notmuch-reindex.c   \
notmuch-reply.c \
notmuch-restore.c   \
notmuch-search.c\
diff --git a/doc/conf.py b/doc/conf.py
index a3d82696..aa864b3c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -95,6 +95,10 @@ man_pages = [
  u'incorporate new mail into the notmuch database',
  [notmuch_authors], 1),
 
+('man1/notmuch-reindex', 'notmuch-reindex',
+ u're-index matching messages',
+ [notmuch_authors], 1),
+
 ('man1/notmuch-reply', 'notmuch-reply',
  u'constructs a reply template for a set of messages',
  [notmuch_authors], 1),
diff --git a/doc/index.rst b/doc/index.rst
index 344606d9..aa6c9f40 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -18,6 +18,7 @@ Contents:
man5/notmuch-hooks
man1/notmuch-insert
man1/notmuch-new
+   man1/notmuch-reindex
man1/notmuch-reply
man1/notmuch-restore
man1/notmuch-search
diff --git a/doc/man1/notmuch-reindex.rst b/doc/man1/notmuch-reindex.rst
new file mode 100644
index ..e39cc4ee
--- /dev/null
+++ b/doc/man1/notmuch-reindex.rst
@@ -0,0 +1,29 @@
+===
+notmuch-reindex
+===
+
+SYNOPSIS
+
+
+**notmuch** **reindex** [*option* ...] <*search-term*> ...
+
+DESCRIPTION
+===
+
+Re-index all messages matching the search terms.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<*search-term*\ >.
+
+The **reindex** command searches for all messages matching the
+supplied search terms, and re-creates the full-text index on these
+messages using the supplied options.
+
+SEE ALSO
+
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch.rst b/doc/man1/notmuch.rst
index fbd7f381..b2a8376e 100644
--- a/doc/man1/notmuch.rst
+++ b/doc/man1/notmuch.rst
@@ -149,8 +149,8 @@ SEE ALSO
 
 **notmuch-address(1)**, **notmuch-compact(1)**, **notmuch-config(1)**,
 **notmuch-count(1)**, **notmuch-dump(1)**, **notmuch-hooks(5)**,
-**notmuch-insert(1)**, **notmuch-new(1)**, **notmuch-reply(1)**,
-**notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-insert(1)**, **notmuch-new(1)**, **notmuch-reindex(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
 **notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
 
 The notmuch website: **https://notmuchmail.org**
diff --git a/doc/man7/notmuch-search-terms.rst 
b/doc/man7/notmuch-search-terms.rst
index 47cab48d..dd76972e 100644
--- a/doc/man7/notmuch-search-terms.rst
+++ b/doc/man7/notmuch-search-terms.rst
@@ -9,6 +9,8 @@ SYNOPSIS
 
 **notmuch** **dump** [--format=(batch-tag|sup)] [--] [--output=<*file*>] [--] 
[<*search-term*> ...]
 
+**notmuch** **reindex** [option ...] <*search-term*> ...
+
 **notmuch** **search** [option ...] <*search-term*> ...
 
 **notmuch** **show** [option ...] <*search-term*> ...
@@ -421,5 +423,6 @@ SEE ALSO
 
 **notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
 **notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
-**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
-**notmuch-search(1)**, **notmuch-show(1)**, **notmuch-tag(1)**
+**notmuch-new(1)**, **notmuch-reindex(1)**, **notmuch-reply(1)**,
+**notmuch-restore(1)**, **notmuch-search(1)**, **notmuch-show(1)**,
+**notmuch-tag(1)**
diff --git a/notmuch-client.h b/notmuch-client.h
index a6f70eae..ab7138c6 100644
--- a/notmuch-client.h
+++ 

Semi-ready saved search

[Tomas Nordin]

Hello Notmuchers

What is a good way (with emacs) to hook into the search to modify only a
part of a search term. I find myself often doing this:

date:2d.. not is:rfile

rfile is my tag for mails that i archive locally. I am looking for a way
to query myself only for the number of days in the minibuffer. I use
this search to filter off all mailing list stuff from a search. So, in
the above example I would only have to answer 2 .

Sunny regards
--
Tomas
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 08/12] lib: add notmuch_thread_get_total_files

[David Bremner]

This is relatively inexpensive in terms of run time and implementation
cost as we are already traversing the list of messages in a thread.
---
 lib/notmuch.h | 12 
 lib/thread.cc |  9 +
 2 files changed, 21 insertions(+)

diff --git a/lib/notmuch.h b/lib/notmuch.h
index ed7da49a..86ad489f 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1095,6 +1095,18 @@ int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 
 /**
+ * Get the total number of files in 'thread'.
+ *
+ * This sums notmuch_message_count_files over all messages in the
+ * thread
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
+/**
  * Get a notmuch_messages_t iterator for the top-level messages in
  * 'thread' in oldest-first order.
  *
diff --git a/lib/thread.cc b/lib/thread.cc
index 561ca5be..d385102b 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -44,6 +44,7 @@ struct visible _notmuch_thread {
 
 GHashTable *message_hash;
 int total_messages;
+int total_files;
 int matched_messages;
 time_t oldest;
 time_t newest;
@@ -266,6 +267,7 @@ _thread_add_message (notmuch_thread_t *thread,
 _notmuch_message_list_add_message (thread->message_list,
   talloc_steal (thread, message));
 thread->total_messages++;
+thread->total_files += notmuch_message_count_files (message);
 
 g_hash_table_insert (thread->message_hash,
 xstrdup (notmuch_message_get_message_id (message)),
@@ -495,6 +497,7 @@ _notmuch_thread_create (void *ctx,
  free, NULL);
 
 thread->total_messages = 0;
+thread->total_files = 0;
 thread->matched_messages = 0;
 thread->oldest = 0;
 thread->newest = 0;
@@ -567,6 +570,12 @@ notmuch_thread_get_total_messages (notmuch_thread_t 
*thread)
 }
 
 int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+return thread->total_files;
+}
+
+int
 notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
 {
 return thread->matched_messages;
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 05/12] test: add known broken tests for duplicate message id

[David Bremner]

There are many other problems that could be tested, but these ones we
have some hope of fixing because it doesn't require UI changes, just
indexing changes.
---
 test/T670-duplicate-mid.sh | 28 
 1 file changed, 28 insertions(+)
 create mode 100755 test/T670-duplicate-mid.sh

diff --git a/test/T670-duplicate-mid.sh b/test/T670-duplicate-mid.sh
new file mode 100755
index ..ced28a21
--- /dev/null
+++ b/test/T670-duplicate-mid.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+test_description="duplicate message ids"
+. ./test-lib.sh || exit 1
+
+add_message '[id]="duplicate"' '[subject]="message 1" [filename]=copy1'
+add_message '[id]="duplicate"' '[subject]="message 2" [filename]=copy2'
+
+test_begin_subtest 'Search for second subject'
+test_subtest_known_broken
+cat  
OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+add_message '[id]="duplicate"' '[body]="sekrit" [filename]=copy3'
+test_begin_subtest 'search for body in duplicate file'
+test_subtest_known_broken
+cat  OUTPUT
+test_expect_equal_file EXPECTED OUTPUT
+
+test_done
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 10/12] lib: add _notmuch_message_remove_indexed_terms

[David Bremner]

Testing will be provided via use in notmuch_message_reindex
---
 lib/message.cc| 54 +++
 lib/notmuch-private.h |  2 ++
 2 files changed, 56 insertions(+)

diff --git a/lib/message.cc b/lib/message.cc
index 2d67f6ea..106b767a 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -599,6 +599,59 @@ _notmuch_message_remove_terms (notmuch_message_t *message, 
const char *prefix)
 }
 }
 
+
+/* Remove all terms generated by indexing, i.e. not tags or
+ * properties, along with any automatic tags*/
+notmuch_private_status_t
+_notmuch_message_remove_indexed_terms (notmuch_message_t *message)
+{
+Xapian::TermIterator i;
+
+const std::string
+   id_prefix = _find_prefix ("id"),
+   property_prefix = _find_prefix ("property"),
+   tag_prefix = _find_prefix ("tag"),
+   type_prefix = _find_prefix ("type");
+
+for (i = message->doc.termlist_begin ();
+i != message->doc.termlist_end (); i++) {
+
+   const std::string term = *i;
+
+   if (term.compare (0, type_prefix.size (), type_prefix) == 0)
+   continue;
+
+   if (term.compare (0, id_prefix.size (), id_prefix) == 0)
+   continue;
+
+   if (term.compare (0, property_prefix.size (), property_prefix) == 0)
+   continue;
+
+   if (term.compare (0, tag_prefix.size (), tag_prefix) == 0 &&
+   term.compare (1, strlen("encrypted"), "encrypted") != 0 &&
+   term.compare (1, strlen("signed"), "signed") != 0 &&
+   term.compare (1, strlen("attachment"), "attachment") != 0)
+   continue;
+
+   try {
+   message->doc.remove_term ((*i));
+   message->modified = TRUE;
+   } catch (const Xapian::InvalidArgumentError) {
+   /* Ignore failure to remove non-existent term. */
+   } catch (const Xapian::Error ) {
+   notmuch_database_t *notmuch = message->notmuch;
+
+   if (!notmuch->exception_reported) {
+   _notmuch_database_log(_notmuch_message_database (message), "A 
Xapian exception occurred creating message: %s\n",
+ error.get_msg().c_str());
+   notmuch->exception_reported = TRUE;
+   }
+   return NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+   }
+}
+return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
 /* Return true if p points at "new" or "cur". */
 static bool is_maildir (const char *p)
 {
@@ -646,6 +699,7 @@ _notmuch_message_add_folder_terms (notmuch_message_t 
*message,
 
 talloc_free (folder);
 
+message->modified = TRUE;
 return NOTMUCH_STATUS_SUCCESS;
 }
 
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 69179177..ebfba35d 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -534,6 +534,8 @@ _notmuch_message_add_reply (notmuch_message_t *message,
 notmuch_database_t *
 _notmuch_message_database (notmuch_message_t *message);
 
+void
+_notmuch_message_remove_unprefixed_terms (notmuch_message_t *message);
 /* sha1.c */
 
 char *
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 04/12] lib: refactor notmuch_database_add_message header parsing

[David Bremner]

This function is large and hard to understand and modify. Start to
break it down into meaningful pieces.
---
 lib/add-message.cc| 54 +++-
 lib/message-file.c| 86 +++
 lib/notmuch-private.h | 11 +++
 3 files changed, 101 insertions(+), 50 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 314016a8..2922eaa9 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -468,7 +468,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 notmuch_private_status_t private_status;
 notmuch_bool_t is_ghost = FALSE, is_new = FALSE;
 
-const char *date, *header;
+const char *date;
 const char *from, *to, *subject;
 char *message_id = NULL;
 
@@ -489,57 +489,12 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 if (ret)
goto DONE;
 
-/* Parse message up front to get better error status. */
-ret = _notmuch_message_file_parse (message_file);
+ret = _notmuch_message_file_get_headers (message_file,
+, , , ,
+_id);
 if (ret)
goto DONE;
 
-/* Before we do any real work, (especially before doing a
- * potential SHA-1 computation on the entire file's contents),
- * let's make sure that what we're looking at looks like an
- * actual email message.
- */
-from = _notmuch_message_file_get_header (message_file, "from");
-subject = _notmuch_message_file_get_header (message_file, "subject");
-to = _notmuch_message_file_get_header (message_file, "to");
-
-if ((from == NULL || *from == '\0') &&
-   (subject == NULL || *subject == '\0') &&
-   (to == NULL || *to == '\0')) {
-   ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
-   goto DONE;
-}
-
-/* Now that we're sure it's mail, the first order of business
- * is to find a message ID (or else create one ourselves).
- */
-header = _notmuch_message_file_get_header (message_file, "message-id");
-if (header && *header != '\0') {
-   message_id = _parse_message_id (message_file, header, NULL);
-
-   /* So the header value isn't RFC-compliant, but it's
-* better than no message-id at all.
-*/
-   if (message_id == NULL)
-   message_id = talloc_strdup (message_file, header);
-}
-
-if (message_id == NULL ) {
-   /* No message-id at all, let's generate one by taking a
-* hash over the file's contents.
-*/
-   char *sha1 = _notmuch_sha1_of_file (filename);
-
-   /* If that failed too, something is really wrong. Give up. */
-   if (sha1 == NULL) {
-   ret = NOTMUCH_STATUS_FILE_ERROR;
-   goto DONE;
-   }
-
-   message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
-   free (sha1);
-}
-
 try {
/* Now that we have a message ID, we get a message object,
 * (which may or may not reference an existing document in the
@@ -579,7 +534,6 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
if (ret)
goto DONE;
 
-   date = _notmuch_message_file_get_header (message_file, "date");
_notmuch_message_set_header_values (message, date, from, subject);
 
ret = _notmuch_message_index_file (message, message_file);
diff --git a/lib/message-file.c b/lib/message-file.c
index db18b163..70526ef0 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -92,6 +92,12 @@ _notmuch_message_file_open (notmuch_database_t *notmuch,
 return _notmuch_message_file_open_ctx (notmuch, NULL, filename);
 }
 
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message_file)
+{
+return message_file->filename;
+}
+
 void
 _notmuch_message_file_close (notmuch_message_file_t *message)
 {
@@ -304,3 +310,83 @@ _notmuch_message_file_get_header (notmuch_message_file_t 
*message,
 
 return decoded;
 }
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+  const char **from_out,
+  const char **subject_out,
+  const char **to_out,
+  const char **date_out,
+  char **message_id_out)
+{
+notmuch_status_t ret;
+const char *header;
+const char *from, *to, *subject, *date;
+char *message_id = NULL;
+
+/* Parse message up front to get better error status. */
+ret = _notmuch_message_file_parse (message_file);
+if (ret)
+   goto DONE;
+
+/* Before we do any real work, (especially before doing a
+ * potential SHA-1 computation on the entire file's contents),
+ * let's make sure that what we're looking at looks like an
+ * actual email message.
+ */
+from = _notmuch_message_file_get_header (message_file, "from");
+

BUG: notmuch dump --include=properties implies --include=tags

[David Bremner]

This is almost certainly my fault, but

% notmuch dump --include=properties '*'

also includes the (typically much more voluminous) tag information.

I looks like the call to dump_tags_message on line 253 needs to be
guarded by (include & DUMP_INCLUDE_TAGS).

I'm not sure how serious a bug this is, e.g. if it's worth fixing for
debian stretch.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 09/12] cli/search: print total number of files matched in summary output.

[David Bremner]

This change needs to be documented. Left for a future commit as the
current format is only loosely documented.
---
 notmuch-search.c | 15 +--
 test/T080-search.sh  |  2 +-
 test/T100-search-by-folder.sh|  4 ++--
 test/T340-maildir-sync.sh|  4 ++--
 test/T370-search-folder-coherence.sh |  2 +-
 test/T500-search-date.sh |  2 +-
 6 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/notmuch-search.c b/notmuch-search.c
index 019e14ee..380e9d8f 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -160,6 +160,7 @@ do_search_threads (search_context_t *ctx)
const char *subject = notmuch_thread_get_subject (thread);
const char *thread_id = notmuch_thread_get_thread_id (thread);
int matched = notmuch_thread_get_matched_messages (thread);
+   int files = notmuch_thread_get_total_files (thread);
int total = notmuch_thread_get_total_messages (thread);
const char *relative_date = NULL;
notmuch_bool_t first_tag = TRUE;
@@ -175,13 +176,23 @@ do_search_threads (search_context_t *ctx)
 
if (format->is_text_printer) {
 /* Special case for the text formatter */
-   printf ("thread:%s %12s [%d/%d] %s; %s (",
+   printf ("thread:%s %12s ",
thread_id,
-   relative_date,
+   relative_date);
+   if (total == files)
+   printf ("[%d/%d] %s; %s (",
matched,
total,
sanitize_string (ctx_quote, authors),
sanitize_string (ctx_quote, subject));
+   else
+   printf ("[%d/%d(%d)] %s; %s (",
+   matched,
+   total,
+   files,
+   sanitize_string (ctx_quote, authors),
+   sanitize_string (ctx_quote, subject));
+
} else { /* Structured Output */
format->map_key (format, "thread");
format->string (format, thread_id);
diff --git a/test/T080-search.sh b/test/T080-search.sh
index d2d71ca9..3bb3dced 100755
--- a/test/T080-search.sh
+++ b/test/T080-search.sh
@@ -111,7 +111,7 @@ thread:XXX   2009-11-18 [3/3] Adrian Perez de Castro, Keith 
Packard, Carl Worth;
 thread:XXX   2009-11-18 [3/3] Israel Herraiz, Keith Packard, Carl Worth; 
[notmuch] New to the list (inbox unread)
 thread:XXX   2009-11-18 [3/3] Jan Janak, Carl Worth; [notmuch] What a great 
idea! (inbox unread)
 thread:XXX   2009-11-18 [2/2] Jan Janak, Carl Worth; [notmuch] [PATCH] Older 
versions of install do not support -C. (inbox unread)
-thread:XXX   2009-11-18 [3/3] Aron Griffis, Keith Packard, Carl Worth; 
[notmuch] archive (inbox unread)
+thread:XXX   2009-11-18 [3/3(4)] Aron Griffis, Keith Packard, Carl Worth; 
[notmuch] archive (inbox unread)
 thread:XXX   2009-11-18 [2/2] Keith Packard, Carl Worth; [notmuch] [PATCH] 
Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox 
unread)
 thread:XXX   2009-11-18 [7/7] Lars Kellogg-Stedman, Mikhail Gusarov, Keith 
Packard, Carl Worth; [notmuch] Working with Maildir storage? (inbox signed 
unread)
 thread:XXX   2009-11-18 [5/5] Mikhail Gusarov, Carl Worth, Keith Packard; 
[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox 
unread)
diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh
index 2844ec61..79c266e4 100755
--- a/test/T100-search-by-folder.sh
+++ b/test/T100-search-by-folder.sh
@@ -15,7 +15,7 @@ add_message '[dir]=things/bad' '[subject]="Bites, stings, sad 
feelings"'
 test_begin_subtest "Single-world folder: specification (multiple results)"
 output=$(notmuch search folder:bad folder:bad/news folder:things/bad | 
notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; 
To the bone (inbox unread)
-thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)
+thread:XXX   2001-01-05 [1/1(2)] Notmuch Test Suite; Bears (inbox unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings 
(inbox unread)"
 
 test_begin_subtest "Top level folder"
@@ -24,7 +24,7 @@ test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] 
Notmuch Test Suite; T
 
 test_begin_subtest "Two-word path to narrow results to one"
 output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
-test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; 
Bears (inbox unread)"
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1(2)] Notmuch Test 
Suite; Bears (inbox unread)"
 
 test_begin_subtest "Folder search with --output=files"
 output=$(notmuch search --output=files folder:bad/news | 
notmuch_search_files_sanitize)
diff --git a/test/T340-maildir-sync.sh b/test/T340-maildir-sync.sh
index 

Re: v3 of regexp search for mid/folder/path

[David Bremner]

David Bremner  writes:

> No sooner posted than I realized it had a bug: the previous version
> compared against the prefixed term so anchored searches failed.
>
> I've also included some tests for the new features in this version.
>
> Below is an interdiff against v1

Gauteh reported success with these patches on IRC. Anyone want more time
to review?

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 11/12] lib: add notmuch_message_reindex

[David Bremner]

From: Daniel Kahn Gillmor 

This new function asks the database to reindex a given message.
The parameter `indexopts` is currently ignored, but is intended to
provide an extensible API to support e.g. changing the encryption or
filtering status (e.g. whether and how certain non-plaintext parts are
indexed).
---
 lib/add-message.cc|   2 +-
 lib/message.cc| 108 +-
 lib/notmuch-private.h |   6 +++
 lib/notmuch.h |  15 +++
 4 files changed, 129 insertions(+), 2 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index ae9b14a7..26405742 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -220,7 +220,7 @@ _my_talloc_free_for_g_hash (void *ptr)
 talloc_free (ptr);
 }
 
-static notmuch_status_t
+notmuch_status_t
 _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
   notmuch_message_t *message,
   notmuch_message_file_t *message_file,
diff --git a/lib/message.cc b/lib/message.cc
index 106b767a..c0e2e803 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -579,7 +579,9 @@ void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
 Xapian::TermIterator i;
-size_t prefix_len = strlen (prefix);
+size_t prefix_len = 0;
+
+prefix_len = strlen (prefix);
 
 while (1) {
i = message->doc.termlist_begin ();
@@ -1934,3 +1936,107 @@ _notmuch_message_frozen (notmuch_message_t *message)
 {
 return message->frozen;
 }
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+notmuch_param_t unused (*indexopts))
+{
+notmuch_database_t *notmuch = NULL;
+notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+notmuch_private_status_t private_status;
+notmuch_filenames_t *orig_filenames = NULL;
+const char *orig_thread_id = NULL;
+notmuch_message_file_t *message_file = NULL;
+
+int found = 0;
+
+if (message == NULL)
+   return NOTMUCH_STATUS_NULL_POINTER;
+
+/* Save in case we need to delete message */
+orig_thread_id = notmuch_message_get_thread_id (message);
+if (!orig_thread_id) {
+   /* XXX TODO: make up new error return? */
+   INTERNAL_ERROR ("message without thread-id");
+}
+
+/* strdup it because the metadata may be invalidated */
+orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+notmuch = _notmuch_message_database (message);
+
+ret = _notmuch_database_ensure_writable (notmuch);
+if (ret)
+   return ret;
+
+orig_filenames = notmuch_message_get_filenames (message);
+
+private_status = _notmuch_message_remove_indexed_terms (message);
+if (private_status) {
+   ret = COERCE_STATUS(private_status, "error removing terms");
+   goto DONE;
+}
+
+/* re-add the filenames with the associated indexopts */
+for (; notmuch_filenames_valid (orig_filenames);
+notmuch_filenames_move_to_next (orig_filenames)) {
+
+   const char *date;
+   const char *from, *to, *subject;
+   char *message_id = NULL;
+   const char *thread_id = NULL;
+
+   const char *filename = notmuch_filenames_get (orig_filenames);
+
+   message_file = _notmuch_message_file_open (notmuch, filename);
+   if (message_file == NULL)
+   continue;
+
+   ret = _notmuch_message_file_get_headers (message_file,
+, , , ,
+_id);
+   if (ret)
+   goto DONE;
+
+   /* XXX TODO: deal with changing message id? */
+
+   _notmuch_message_add_filename (message, filename);
+
+   ret = _notmuch_database_link_message_to_parents (notmuch, message,
+message_file,
+_id);
+   if (ret)
+   goto DONE;
+
+   if (thread_id == NULL)
+   thread_id = orig_thread_id;
+
+   _notmuch_message_add_term (message, "thread", thread_id);
+   _notmuch_message_set_header_values (message, date, from, subject);
+
+   ret = _notmuch_message_index_file (message, message_file);
+
+   if (ret == NOTMUCH_STATUS_FILE_ERROR)
+   continue;
+   if (ret)
+   goto DONE;
+
+   found++;
+   _notmuch_message_file_close (message_file);
+   message_file = NULL;
+}
+if (found == 0) {
+   /* put back thread id to help cleanup */
+   _notmuch_message_add_term (message, "thread", orig_thread_id);
+   ret = _notmuch_message_delete (message);
+} else {
+   _notmuch_message_sync (message);
+}
+
+ DONE:
+if (message_file)
+   _notmuch_message_file_close (message_file);
+
+/* XXX TODO destroy orig_filenames? */
+return ret;
+}
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index ebfba35d..beb50033 

[PATCH 07/12] lib: add notmuch_message_count_files

[David Bremner]

This operation is relatively inexpensive, as the needed metadata is
already computed by our lazy metadata fetching. The goal is to support
better UI for messages with multipile files.
---
 lib/message.cc| 8 
 lib/notmuch-private.h | 6 ++
 lib/notmuch.h | 8 
 lib/string-list.c | 6 ++
 4 files changed, 28 insertions(+)

diff --git a/lib/message.cc b/lib/message.cc
index c2721191..2d67f6ea 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -946,6 +946,14 @@ notmuch_message_get_filenames (notmuch_message_t *message)
 return _notmuch_filenames_create (message, message->filename_list);
 }
 
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+_notmuch_message_ensure_filename_list (message);
+
+return _notmuch_string_list_length (message->filename_list);
+}
+
 notmuch_bool_t
 notmuch_message_get_flag (notmuch_message_t *message,
  notmuch_message_flag_t flag)
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index f3c058ab..69179177 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -558,6 +558,12 @@ typedef struct visible _notmuch_string_list {
 notmuch_string_list_t *
 _notmuch_string_list_create (const void *ctx);
 
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
 /* Add 'string' to 'list'.
  *
  * The list will create its own talloced copy of 'string'.
diff --git a/lib/notmuch.h b/lib/notmuch.h
index d374dc96..ed7da49a 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -1340,6 +1340,14 @@ notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
 /**
+ * Get the total number of files associated with a message.
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
+/**
  * Get a filename for the email corresponding to 'message'.
  *
  * The returned filename is an absolute filename, (the initial
diff --git a/lib/string-list.c b/lib/string-list.c
index 43ebe499..9c3ae7ef 100644
--- a/lib/string-list.c
+++ b/lib/string-list.c
@@ -42,6 +42,12 @@ _notmuch_string_list_create (const void *ctx)
 return list;
 }
 
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+return list->length;
+}
+
 void
 _notmuch_string_list_append (notmuch_string_list_t *list,
 const char *string)
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

v2 of index multiple files per msg-id, add reindex command

[David Bremner]

This obsoletes id:20170414025004.5334-1-da...@tethera.net. Let me
repeat the warning from that version:

,
| WARNING: reindexing is an intrusive operation. I don't think this will
| corrupt your database, but previous versions thrashed threading pretty
| well. notmuch-dump is your friend.
`

Compared to the previous version, this one cleans up the code to print
total file counts (splitting into multiple commits), and updates doc
and tests.

There is a lot of code movement here, mainly splitting up database.cc

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 02/12] lib/n_d_add_message: refactor test for new/ghost messages

[David Bremner]

The switch is easier to understand than the side effects in the if
test. It also potentially allows us more flexibility in breaking up
this function into smaller pieces, since passing private_status around
is icky.
---
 lib/add-message.cc | 23 +--
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/add-message.cc b/lib/add-message.cc
index 5fe2c45b..0f09415e 100644
--- a/lib/add-message.cc
+++ b/lib/add-message.cc
@@ -570,7 +570,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 notmuch_message_t *message = NULL;
 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
 notmuch_private_status_t private_status;
-notmuch_bool_t is_ghost = false;
+notmuch_bool_t is_ghost = FALSE, is_new = FALSE;
 
 const char *date, *header;
 const char *from, *to, *subject;
@@ -655,7 +655,17 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
talloc_free (message_id);
 
-   if (message == NULL) {
+   /* We cannot call notmuch_message_get_flag for a new message */
+   switch (private_status) {
+   case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+   is_ghost = FALSE;
+   is_new = TRUE;
+   break;
+   case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+   is_ghost = notmuch_message_get_flag (message, 
NOTMUCH_MESSAGE_FLAG_GHOST);
+   is_new = FALSE;
+   break;
+   default:
ret = COERCE_STATUS (private_status,
 "Unexpected status value from 
_notmuch_message_create_for_message_id");
goto DONE;
@@ -663,18 +673,11 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
_notmuch_message_add_filename (message, filename);
 
-   /* Is this a newly created message object or a ghost
-* message?  We have to be slightly careful: if this is a
-* blank message, it's not safe to call
-* notmuch_message_get_flag yet. */
-   if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND ||
-   (is_ghost = notmuch_message_get_flag (
-   message, NOTMUCH_MESSAGE_FLAG_GHOST))) {
+   if (is_new || is_ghost) {
_notmuch_message_add_term (message, "type", "mail");
if (is_ghost)
/* Convert ghost message to a regular message */
_notmuch_message_remove_term (message, "type", "ghost");
-
ret = _notmuch_database_link_message (notmuch, message,
  message_file, is_ghost);
if (ret)
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/6] lib: add content type argument to uuencode filter.

[David Bremner]

The idea is to support more general types of filtering, based on
content type.
---
 lib/index.cc | 13 -
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 8c145540..1c04cc3d 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -56,6 +56,7 @@ typedef struct _NotmuchFilterDiscardUuencodeClass 
NotmuchFilterDiscardUuencodeCl
  **/
 struct _NotmuchFilterDiscardUuencode {
 GMimeFilter parent_object;
+GMimeContentType *content_type;
 int state;
 };
 
@@ -63,7 +64,7 @@ struct _NotmuchFilterDiscardUuencodeClass {
 GMimeFilterClass parent_class;
 };
 
-static GMimeFilter *notmuch_filter_discard_uuencode_new (void);
+static GMimeFilter *notmuch_filter_discard_uuencode_new (GMimeContentType 
*content);
 
 static void notmuch_filter_discard_uuencode_finalize (GObject *object);
 
@@ -102,8 +103,9 @@ notmuch_filter_discard_uuencode_finalize (GObject *object)
 static GMimeFilter *
 filter_copy (GMimeFilter *gmime_filter)
 {
-(void) gmime_filter;
-return notmuch_filter_discard_uuencode_new ();
+NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
+
+return notmuch_filter_discard_uuencode_new (filter->content_type);
 }
 
 static void
@@ -196,7 +198,7 @@ filter_reset (GMimeFilter *gmime_filter)
  * Returns: a new #NotmuchFilterDiscardUuencode filter.
  **/
 static GMimeFilter *
-notmuch_filter_discard_uuencode_new (void)
+notmuch_filter_discard_uuencode_new (GMimeContentType *content_type)
 {
 static GType type = 0;
 NotmuchFilterDiscardUuencode *filter;
@@ -220,6 +222,7 @@ notmuch_filter_discard_uuencode_new (void)
 
 filter = (NotmuchFilterDiscardUuencode *) g_object_newv (type, 0, NULL);
 filter->state = 0;
+filter->content_type = content_type;
 
 return (GMimeFilter *) filter;
 }
@@ -396,7 +399,7 @@ _index_mime_part (notmuch_message_t *message,
 g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
 filter = g_mime_stream_filter_new (stream);
-discard_uuencode_filter = notmuch_filter_discard_uuencode_new ();
+discard_uuencode_filter = notmuch_filter_discard_uuencode_new 
(content_type);
 
 g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
  discard_uuencode_filter);
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/2] cli/dump: don't include tags when not asked for

[David Bremner]

Add in the analogous test for tags that is given for properties a few
lines below.
---
 notmuch-dump.c| 3 ++-
 test/T610-message-property.sh | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/notmuch-dump.c b/notmuch-dump.c
index e451ac0c..5cc3b2f6 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -250,7 +250,8 @@ database_dump_file (notmuch_database_t *notmuch, gzFile 
output,
 
message = notmuch_messages_get (messages);
 
-   if (dump_tags_message (notmuch, message, output_format, output,
+   if ((include & DUMP_INCLUDE_TAGS) &&
+   dump_tags_message (notmuch, message, output_format, output,
   , _size))
return EXIT_FAILURE;
 
diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh
index 7d95bde6..ba5f55da 100755
--- a/test/T610-message-property.sh
+++ b/test/T610-message-property.sh
@@ -210,7 +210,6 @@ notmuch dump | grep '^#=' > OUTPUT
 test_expect_equal_file PROPERTIES OUTPUT
 
 test_begin_subtest "dump _only_ message properties"
-test_subtest_known_broken
 cat < EXPECTED
 #notmuch-dump batch-tag:3 properties
 #= 4efc743a.3060...@april.org 
fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice 
testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 
testkey3=bob3 testkey3=testvalue3
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] emacs: tree: bugfix: specify --format-version

[David Bremner]

Mark Walters  writes:

> Previously notmuch tree did not specify the format-version when
> calling notmuch. This meant that when the structured output was
> slightly changed (in commit 14c60cf168ac3b0f277188c16e6012b7ebdadde7)
> stash filename broke. This fixes this breakage by specifying the
> format-version.
> ---
>
> Bremner pointed out this bug today on irc. In due course we may want
> to use format-version=3 and update the helper functions but as none of
> notmuch-emacs uses the new format version yet this seem the right fix for now.

Pushed to release and master.

collecting bug fixes for a potential 0.24.2

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

synchronizing the inbox flag

[Antoine Beaupré]

Hi notmuchers! :)

I've done numerous attemps at making notmuch work across different
machines in the past. (See below for the details.) The only thing
missing now is the "inbox" tag synchronization. My use case is this: I
mostly use this one machine all the time. I have a [notmuch-tag][1]
script that sets a number of tags on messages depending on the folder
they're in and so on, so most tags are actually static and basically
shortcuts for saved searches.

Except the inbox tag: that tag is "floating" in the notmuch database and
is not sync'd on the server, which means when i pickup that laptop for
my next travel, i need to resync the whole notmuch database with notmuch
dump / restore. That is error-prone and annoying, and obviously fails
often because I don't regularly perform that sync.

What I would like to see in notmuch would be a way for the inbox tag to
be sync'd to the maildir storage. It could be an extra arbitrary flag in
the maildir filenames. From what I can tell from the [maildir
standard][2], there are 26 possible such such flags, with 6 formally
defined: P (passed), R (replied), S (seen, mapped as unread), F
(flagged), D (draft), T (trashed). The latter is not sync'd in notmuch,
the others are sync'd if synchronized_tags is true (default) in the
config file.

I would propose using the `A` or `I` flag, and sync it with the `inbox`
tag the same way the `S` flag is sync'd with the `unread` tag (ie. add
the flag when `inbox` is removed). Ideally, we could let the user
customize which flags are synchronized with which tags in the
configuration file. Given the limitations of the config file format
(ini-style), I'm not sure how that could be implemented.

Also note that certain IMAP servers treat non-standard flags as IMAP
keywords. [Dovecot][4], in particular, allows users to modify those
mappings by editing the `dovecot-keywords` file. In my local
configuration, where I have not manually edited that file, its contents
are:

0 Junk
1 NonJunk
2 $Forwarded

This is presumably something that Thunderbird did at some point, or it
could be the default Dovecot configuration. So this may mean letters,
`A-C` are already taken, which means there could be interoperability
issues with using `A`: it could mean we tag all inbox messages as junk!
In this case, the `I` flag may be more appropriate (with a meaning the
opposite of `A`, ie. sync'd directly with the inbox tag).

Since there is a considerable range of foot-shooting potential here, it
may be better to just add an extra flag without allowing users to
customize the flags on their own. But I like the idea of allowing that
flexibility: it could fix a bunch of other use cases (like deleting mail
and syncing junk flags as well).

Anyone else working on this? Ideas of how this could be implemented?

Some more historical context follows...

My first attempt was a very old patch (2011!) to add resume
support in Notmuch - something that thankfully landed 6 years later:

1310808075-787-1-git-send-email-anar...@koumbit.org

My second attempt was the synchronization of the "deleted" tag to the T
(trashed) IMAP/maildir flag (~2012):

1310874973-28437-1-git-send-email-anar...@koumbit.org

That also failed and I ended up setting up a [script][3] to delete
messages tagged "deleted" automatically. The script also synchronizes
ham/spam flags.

Finally, I have also experimented with [muchsync][5] but that basically
failed for me: the thing would just run out of ram and fail to
synchronize tags.

[1]: https://gitlab.com/anarcat/scripts/blob/master/notmuch-tag
[2]: https://cr.yp.to/proto/maildir.html
[3]: https://gitlab.com/anarcat/scripts/blob/master/notmuch-purge
[4]: https://wiki2.dovecot.org/MailboxFormat/Maildir#IMAP_keywords
[5]: http://www.muchsync.org/

Thanks for any feedback!

-- 
Blind respect for authority is the greatest enemy of truth.
   - Albert Einstein
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] emacs: with prefix argument, notmuch-show-stash-date stashes timestamp

[Jani Nikula]

On Thu, 11 May 2017, David Bremner  wrote:
> Tomi Ollila  writes:
>
>> Using timestamp of a message is useful in many Xapian queries.
>> ---
>>
>> This is my suggested alternative to id:20170110181525.18269-1-j...@nikula.org
>> since my comments in id:m2fuimv4mj@guru.guru-group.fi
>
> In case it's not obvious, I'm waiting for you two to come to some
> consensus on this one.

This patch seems like a worthwhile addition no matter what, and doesn't
prevent us from adding formatted stashing later if we so decide.

BR,
Jani.
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: [PATCH] emacs: with prefix argument, notmuch-show-stash-date stashes timestamp

[David Bremner]

Tomi Ollila  writes:

> Using timestamp of a message is useful in many Xapian queries.
> ---
>
> This is my suggested alternative to id:20170110181525.18269-1-j...@nikula.org
> since my comments in id:m2fuimv4mj@guru.guru-group.fi

In case it's not obvious, I'm waiting for you two to come to some
consensus on this one.

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 3/6] lib/index: separate state table definition from scanner.

[David Bremner]

We want to reuse the scanner definition with a different table
---
 lib/index.cc | 81 +++-
 1 file changed, 47 insertions(+), 34 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 74a750b9..02b35b81 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -31,6 +31,15 @@ typedef struct _NotmuchFilterDiscardUuencodeClass 
NotmuchFilterDiscardUuencodeCl
 
 typedef void (*filter_fun) (GMimeFilter *filter, char *in, size_t len, size_t 
prespace,
char **out, size_t *outlen, size_t *outprespace);
+
+typedef struct {
+int state;
+int a;
+int b;
+int next_if_match;
+int next_if_not_match;
+} scanner_state_t;
+
 /**
  * NotmuchFilterDiscardUuencode:
  *
@@ -119,46 +128,18 @@ filter_filter (GMimeFilter *gmime_filter, char *inbuf, 
size_t inlen, size_t pres
 }
 
 static void
-filter_filter_uuencode (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, 
size_t prespace,
-   char **outbuf, size_t *outlen, size_t *outprespace)
+do_filter (const scanner_state_t states[],
+  int first_skipping_state,
+  GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t 
prespace,
+  char **outbuf, size_t *outlen, size_t *outprespace)
 {
 NotmuchFilterDiscardUuencode *filter = (NotmuchFilterDiscardUuencode *) 
gmime_filter;
 register const char *inptr = inbuf;
 const char *inend = inbuf + inlen;
 char *outptr;
-
+int next;
 (void) prespace;
 
-/* Simple, linear state-transition diagram for our filter.
- *
- * If the character being processed is within the range of [a, b]
- * for the current state then we transition next_if_match
- * state. If not, we transition to the next_if_not_match state.
- *
- * The final two states are special in that they are the states in
- * which we discard data. */
-static const struct {
-   int state;
-   int a;
-   int b;
-   int next_if_match;
-   int next_if_not_match;
-} states[] = {
-   {0,  'b',  'b',  1,  0},
-   {1,  'e',  'e',  2,  0},
-   {2,  'g',  'g',  3,  0},
-   {3,  'i',  'i',  4,  0},
-   {4,  'n',  'n',  5,  0},
-   {5,  ' ',  ' ',  6,  0},
-   {6,  '0',  '7',  7,  0},
-   {7,  '0',  '7',  8,  0},
-   {8,  '0',  '7',  9,  0},
-   {9,  ' ',  ' ',  10, 0},
-   {10, '\n', '\n', 11, 10},
-   {11, 'M',  'M',  12, 0},
-   {12, ' ',  '`',  12, 11}
-};
-int next;
 
 g_mime_filter_set_size (gmime_filter, inlen, FALSE);
 outptr = gmime_filter->outbuf;
@@ -174,7 +155,7 @@ filter_filter_uuencode (GMimeFilter *gmime_filter, char 
*inbuf, size_t inlen, si
next = states[filter->state].next_if_not_match;
}
 
-   if (filter->state < 11)
+   if (filter->state < first_skipping_state)
*outptr++ = *inptr;
 
filter->state = next;
@@ -187,6 +168,38 @@ filter_filter_uuencode (GMimeFilter *gmime_filter, char 
*inbuf, size_t inlen, si
 }
 
 static void
+filter_filter_uuencode (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, 
size_t prespace,
+   char **outbuf, size_t *outlen, size_t *outprespace)
+{
+/* Simple, linear state-transition diagram for our filter.
+ *
+ * If the character being processed is within the range of [a, b]
+ * for the current state then we transition next_if_match
+ * state. If not, we transition to the next_if_not_match state.
+ *
+ * The final two states are special in that they are the states in
+ * which we discard data. */
+static const scanner_state_t states[] = {
+   {0,  'b',  'b',  1,  0},
+   {1,  'e',  'e',  2,  0},
+   {2,  'g',  'g',  3,  0},
+   {3,  'i',  'i',  4,  0},
+   {4,  'n',  'n',  5,  0},
+   {5,  ' ',  ' ',  6,  0},
+   {6,  '0',  '7',  7,  0},
+   {7,  '0',  '7',  8,  0},
+   {8,  '0',  '7',  9,  0},
+   {9,  ' ',  ' ',  10, 0},
+   {10, '\n', '\n', 11, 10},
+   {11, 'M',  'M',  12, 0},
+   {12, ' ',  '`',  12, 11}
+};
+
+do_filter(states, 11,
+ gmime_filter, inbuf, inlen, prespace, outbuf, outlen, 
outprespace);
+}
+
+static void
 filter_complete (GMimeFilter *filter, char *inbuf, size_t inlen, size_t 
prespace,
 char **outbuf, size_t *outlen, size_t *outprespace)
 {
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: Limiting number of results in saved searches in Emacs ?

[David Bremner]

Olivier Berger  writes:

> Hi.
>
> I can't find a way to limit the number of results in saved searches in
> Emacs. Mainly to avoid OOM issues.
>
> It seems that --limit=N could help on the command-line.
>
> However, the construction of the saved searches only allow to type search 
> terms
> and no options like --limit.
>
> Would anyone have a better suggestion ?

A simple thing to do would be to limit the date range of the saved
search.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: notmuch-0.24.1: missing header include

[David Bremner]

Thomas Klausner  writes:

> Hi!
>
> On Solaris, notmuch-0.24.1 does not compile because lib/message.cc
> uses index(3) but does not include strings.h.
>
> Please apply the attached patch or a similar one.
>
> Thanks,
>  Thomas

In master we've replaced index(3) with strchr(3). Does that fix your
issue, or is the include still needed?

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch