[PATCH 4/4] lib: index the content-type of the parts of encrypted messages

[Jani Nikula]

From: Daniel Kahn Gillmor 

This is a logical followup to "lib: index the content type of
signature parts", which will make it easier to record the message
structure of all messages.
---
 lib/index.cc| 9 -
 test/T350-crypto.sh | 1 -
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 0beaae62f048..ceb444df60bf 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -409,7 +409,14 @@ _index_mime_part (notmuch_message_t *message,
}
}
if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-   /* Don't index encrypted parts. */
+   /* Don't index encrypted parts, but index their content type. */
+   _index_content_type (message,
+g_mime_multipart_get_part (multipart, i));
+   if ((i != GMIME_MULTIPART_ENCRYPTED_VERSION) &&
+   (i != GMIME_MULTIPART_ENCRYPTED_CONTENT)) {
+   _notmuch_database_log (_notmuch_message_database (message),
+  "Warning: Unexpected extra parts of 
multipart/encrypted.\n");
+   }
continue;
}
_index_mime_part (message,
diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index b8f862560006..53bf4113d622 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -236,7 +236,6 @@ test_expect_success \
 "(mml-attach-file \"TESTATTACHMENT\") (mml-secure-message-encrypt)"'
 
 test_begin_subtest "encrypted part content-type indexing"
-test_subtest_known_broken
 output=$(notmuch search mimetype:multipart/encrypted and 
mimetype:application/pgp-encrypted and mimetype:application/octet-stream | 
notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; 
test encrypted message 001 (encrypted inbox)"
 
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 3/4] lib: index the content type of signature parts

[Jani Nikula]

It's useful (*) to be able to easily find messages with certain types
of signatures. Having the mimetype: prefix searches fail for some
content types is also genuinely surprising (*). Index the content type
of signature parts.

While at it, switch to the gmime convenience constants for content and
signature part indexes.

*) At least for developers of email software!
---
 lib/index.cc| 11 +++
 test/T350-crypto.sh |  1 -
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 64bc92a5b56d..0beaae62f048 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -398,12 +398,15 @@ _index_mime_part (notmuch_message_t *message,
 
for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
-   /* Don't index the signature. */
-   if (i == 1)
+   /* Don't index the signature, but index its content type. */
+   if (i == GMIME_MULTIPART_SIGNED_SIGNATURE) {
+   _index_content_type (message,
+g_mime_multipart_get_part (multipart, 
i));
continue;
-   if (i > 1)
+   } else if (i != GMIME_MULTIPART_SIGNED_CONTENT) {
_notmuch_database_log (_notmuch_message_database (message),
- "Warning: Unexpected extra parts of 
multipart/signed. Indexing anyway.\n");
+  "Warning: Unexpected extra parts of 
multipart/signed. Indexing anyway.\n");
+   }
}
if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
/* Don't index encrypted parts. */
diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index 2f1f9ce4913c..b8f862560006 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -38,7 +38,6 @@ test_expect_success \
 "(mml-secure-message-sign)"'
 
 test_begin_subtest "signed part content-type indexing"
-test_subtest_known_broken
 output=$(notmuch search mimetype:multipart/signed and 
mimetype:application/pgp-signature | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; 
test signed message 001 (inbox signed)"
 
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 1/4] lib: abstract content type indexing

[Jani Nikula]

Make the follow-up change of indexing signature content types
easier. No functional changes.
---
 lib/index.cc | 26 +-
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/index.cc b/lib/index.cc
index 2b98b588d38d..64bc92a5b56d 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -351,6 +351,19 @@ _index_address_list (notmuch_message_t *message,
 }
 }
 
+static void
+_index_content_type (notmuch_message_t *message, GMimeObject *part)
+{
+GMimeContentType *content_type = g_mime_object_get_content_type (part);
+if (content_type) {
+   char *mime_string = g_mime_content_type_to_string (content_type);
+   if (mime_string) {
+   _notmuch_message_gen_terms (message, "mimetype", mime_string);
+   g_free (mime_string);
+   }
+}
+}
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
@@ -361,6 +374,7 @@ _index_mime_part (notmuch_message_t *message,
 GMimeDataWrapper *wrapper;
 GByteArray *byte_array;
 GMimeContentDisposition *disposition;
+GMimeContentType *content_type;
 char *body;
 const char *charset;
 
@@ -370,15 +384,7 @@ _index_mime_part (notmuch_message_t *message,
return;
 }
 
-GMimeContentType *content_type = g_mime_object_get_content_type(part);
-if (content_type) {
-   char *mime_string = g_mime_content_type_to_string(content_type);
-   if (mime_string)
-   {
-   _notmuch_message_gen_terms (message, "mimetype", mime_string);
-   g_free(mime_string);
-   }
-}
+_index_content_type (message, part);
 
 if (GMIME_IS_MULTIPART (part)) {
GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -447,6 +453,8 @@ _index_mime_part (notmuch_message_t *message,
 g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
 
 filter = g_mime_stream_filter_new (stream);
+
+content_type = g_mime_object_get_content_type (part);
 discard_non_term_filter = notmuch_filter_discard_non_term_new 
(content_type);
 
 g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 2/4] test: signed and encrypted part content type indexing

[Jani Nikula]

Add known broken subtests for searching signed and encrypted messages
using mimetype: prefix search for the content-types of signed and
encrypted parts.
---
 test/T350-crypto.sh | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/test/T350-crypto.sh b/test/T350-crypto.sh
index 1d408af71dd7..2f1f9ce4913c 100755
--- a/test/T350-crypto.sh
+++ b/test/T350-crypto.sh
@@ -37,6 +37,11 @@ test_expect_success \
 "This is a test signed message." \
 "(mml-secure-message-sign)"'
 
+test_begin_subtest "signed part content-type indexing"
+test_subtest_known_broken
+output=$(notmuch search mimetype:multipart/signed and 
mimetype:application/pgp-signature | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; 
test signed message 001 (inbox signed)"
+
 test_begin_subtest "signature verification"
 output=$(notmuch show --format=json --verify subject:"test signed message 001" 
\
 | notmuch_json_show_sanitize \
@@ -231,6 +236,11 @@ test_expect_success \
 "This is a test encrypted message.\n" \
 "(mml-attach-file \"TESTATTACHMENT\") (mml-secure-message-encrypt)"'
 
+test_begin_subtest "encrypted part content-type indexing"
+test_subtest_known_broken
+output=$(notmuch search mimetype:multipart/encrypted and 
mimetype:application/pgp-encrypted and mimetype:application/octet-stream | 
notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; 
test encrypted message 001 (encrypted inbox)"
+
 test_begin_subtest "decryption, --format=text"
 output=$(notmuch show --format=text --decrypt subject:"test encrypted message 
001" \
 | notmuch_show_sanitize_all \
-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[PATCH 0/4] lib: signed and encrypted part content-type indexing

[Jani Nikula]

v2 of signed and encrypted part content-type indexing, including both my
patches [1] and Daniel's [2], now with tests. Tests added first to prove
we don't currently index this stuff and the patches make a
difference. No other changes.

BR,
Jani.

[1] id:cover.1505329740.git.j...@nikula.org 
[1] id:20170915053852.23663-1-...@fifthhorseman.net

Daniel Kahn Gillmor (1):
  lib: index the content-type of the parts of encrypted messages

Jani Nikula (3):
  lib: abstract content type indexing
  test: signed and encrypted part content type indexing
  lib: index the content type of signature parts

 lib/index.cc| 46 --
 test/T350-crypto.sh |  8 
 2 files changed, 40 insertions(+), 14 deletions(-)

-- 
2.11.0

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: What is notmuch compact and should I use it?

[Attic Hermit]

All things are clear now. Thank you!

David Bremner  writes:

> Attic Hermit  writes:
>
>> David Bremner  writes:
>>
>>> There is some tradeoff: search should be faster after a compact, but
>>> indexing many messages will likely be slower.
>>
>> Maybe I'm wrong, but is it meaning that it will be taking a long time to
>> compact many messages, or after doing `notmuch compact`, indexing many
>> new messages will be slower because of the compacted database?
>
> The latter, according to the xapian documentation.
>
> d
-- 
Attic Hermit
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: What is notmuch compact and should I use it?

[David Bremner]

Attic Hermit  writes:

> David Bremner  writes:
>
>> There is some tradeoff: search should be faster after a compact, but
>> indexing many messages will likely be slower.
>
> Maybe I'm wrong, but is it meaning that it will be taking a long time to
> compact many messages, or after doing `notmuch compact`, indexing many
> new messages will be slower because of the compacted database?

The latter, according to the xapian documentation.

d
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: What is notmuch compact and should I use it?

[Attic Hermit]

David Bremner  writes:

> There is some tradeoff: search should be faster after a compact, but
> indexing many messages will likely be slower.

Maybe I'm wrong, but is it meaning that it will be taking a long time to
compact many messages, or after doing `notmuch compact`, indexing many
new messages will be slower because of the compacted database?

-- 
Attic Hermit
___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

Re: What is notmuch compact and should I use it?

[David Bremner]

Attic Hermit  writes:

> What exactly is to compact the database? I noticed that the size of the
> notmuch database is shrunk after `notmuch compact`, but I don't think
> this procedure is gzipping the database. I thought it is something like
> the garbage collecting, but I'm not sure.

Yes, that's approximately correct; notmuch compact is essentially
equivalent to xapian compact

 https://xapian.org/docs/admin_notes.html#compacting-a-database
 
> And should I use it regularly? if it is, how often should it be done?

There is some tradeoff: search should be faster after a compact, but
indexing many messages will likely be slower. For most people this
doesn't matter too much since the number of messages added after the
original index is relatively small.

d

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch

[RFC PATCH] emacs: cl-letf enriched-decode-display-prop for text/encriched display

[Tomi Ollila]

Dynamically bind enriched-decode-display-prop when inserting
text/enriched part. This complements commit 9b0582383833 for
emacs versions before 24.4 which do not have advice-add functionality.
---

This is sent as RFC, as I did not (yet) have time to generate/find some
data to test it... anyway to me this looks good (on digital paper ;)

A couple of days ago I spent a little time to find how cl-flet & cl-letf
works -- cl-flet cannot be used as replacement for flet, since former
works in lexical scope... and accorging to this page

http://endlessparentheses.com/understanding-letf-and-how-it-replaces-flet.html

the (cl-letf (((symbol-function 'enriched-decode-display-prop) ...
should do the trick...

(ok, now I grepped notmuch code (again). It seens we're already using
letf & cl-letf and there is no reference to flet ;/ (I thought there were
when I grepped last time) -- well good learning experience...)

Tomi

 emacs/notmuch-show.el | 11 ---
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 99390277..9ebd50ed 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -773,14 +773,11 @@ (defun notmuch-show-insert-part-text/calendar (msg part 
content-type nth depth b
 (defun notmuch-show-insert-part-text/x-vcalendar (msg part content-type nth 
depth button)
   (notmuch-show-insert-part-text/calendar msg part content-type nth depth 
button))
 
-;; https://bugs.gnu.org/28350
-(defun notmuch-show--enriched-decode-display-prop (start end  param)
-  (list start end))
-
 (defun notmuch-show-insert-part-text/enriched (msg part content-type nth depth 
button)
-  (advice-add 'enriched-decode-display-prop :override
- #'notmuch-show--enriched-decode-display-prop)
-  nil)
+  ;; https://bugs.gnu.org/28350
+  (cl-letf (((symbol-function 'enriched-decode-display-prop)
+(lambda (start end  param) (list start end
+(notmuch-show-insert-part-*/* msg part content-type nth depth button)))
 
 (defun notmuch-show-get-mime-type-of-application/octet-stream (part)
   ;; If we can deduce a MIME type from the filename of the attachment,
-- 
2.13.3

___
notmuch mailing list
notmuch@notmuchmail.org
https://notmuchmail.org/mailman/listinfo/notmuch