This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 5ccc59b3e3 JAMES-4052 Add optional user property to OpenSearch index
(#2363)
5ccc59b3e3 is described below
commit 5ccc59b3e3582bf0d8fc86494f187e8f85ea52f8
Author: Benoit TELLIER <[email protected]>
AuthorDate: Fri Jul 26 10:20:43 2024 +0200
JAMES-4052 Add optional user property to OpenSearch index (#2363)
---
.../pages/distributed/configure/opensearch.adoc | 4 +++
.../mailbox/opensearch/MailboxMappingFactory.java | 3 ++
.../opensearch/OpenSearchMailboxConfiguration.java | 35 +++++++++++++++---
.../OpenSearchListeningMessageSearchIndex.java | 4 +--
.../mailbox/opensearch/json/IndexableMessage.java | 23 ++++++++++--
.../opensearch/json/JsonMessageConstants.java | 1 +
.../opensearch/json/MessageToOpenSearchJson.java | 41 +++++++++++++++++++++-
src/site/xdoc/server/config-opensearch.xml | 2 ++
upgrade-instructions.md | 26 +++++++++++++-
9 files changed, 128 insertions(+), 11 deletions(-)
diff --git a/docs/modules/servers/pages/distributed/configure/opensearch.adoc
b/docs/modules/servers/pages/distributed/configure/opensearch.adoc
index 0b70fd3b43..fd061b4407 100644
--- a/docs/modules/servers/pages/distributed/configure/opensearch.adoc
+++ b/docs/modules/servers/pages/distributed/configure/opensearch.adoc
@@ -99,6 +99,10 @@ Default to false.
| opensearch.indexBody
| Indicates if you wish to index body or not (default: true). This can be used
to decrease the performance cost associated with indexing.
+
+| opensearch.indexUser
+| Indicates if you wish to index user or not (default: false). This can be
used to have per user reports in OpenSearch Dashboards.
+
|===
=== Quota search
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java
index 9d97e9fd9e..84a61f9b9b 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java
@@ -251,6 +251,9 @@ public class MailboxMappingFactory {
.put(JsonMessageConstants.MIME_MESSAGE_ID, new Property.Builder()
.keyword(new KeywordProperty.Builder().build())
.build())
+ .put(JsonMessageConstants.USER, new Property.Builder()
+ .keyword(new KeywordProperty.Builder().build())
+ .build())
.put(JsonMessageConstants.TEXT_BODY, new Property.Builder()
.text(new TextProperty.Builder().analyzer(STANDARD).build())
.build())
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
index 6fa68d0286..16958f6411 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
@@ -26,6 +26,7 @@ import org.apache.commons.configuration2.Configuration;
import org.apache.james.backends.opensearch.IndexName;
import org.apache.james.backends.opensearch.ReadAliasName;
import org.apache.james.backends.opensearch.WriteAliasName;
+import
org.apache.james.mailbox.opensearch.json.MessageToOpenSearchJson.IndexUser;
public class OpenSearchMailboxConfiguration {
@@ -37,6 +38,7 @@ public class OpenSearchMailboxConfiguration {
private Optional<IndexHeaders> indexHeaders;
private Optional<Boolean> optimiseMoves;
private Optional<IndexBody> indexBody;
+ private Optional<IndexUser> indexUser;
Builder() {
indexMailboxName = Optional.empty();
@@ -46,6 +48,7 @@ public class OpenSearchMailboxConfiguration {
indexHeaders = Optional.empty();
optimiseMoves = Optional.empty();
indexBody = Optional.empty();
+ indexUser = Optional.empty();
}
public Builder indexMailboxName(Optional<IndexName> indexMailboxName) {
@@ -83,6 +86,11 @@ public class OpenSearchMailboxConfiguration {
return this;
}
+ public Builder indexUser(IndexUser indexUser) {
+ this.indexUser = Optional.ofNullable(indexUser);
+ return this;
+ }
+
public OpenSearchMailboxConfiguration build() {
return new OpenSearchMailboxConfiguration(
indexMailboxName.orElse(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX),
@@ -91,7 +99,8 @@ public class OpenSearchMailboxConfiguration {
indexAttachment.orElse(IndexAttachments.YES),
indexHeaders.orElse(IndexHeaders.YES),
optimiseMoves.orElse(DEFAULT_OPTIMIZE_MOVES),
- indexBody.orElse(IndexBody.YES));
+ indexBody.orElse(IndexBody.YES),
+ indexUser.orElse(IndexUser.NO));
}
}
@@ -109,10 +118,12 @@ public class OpenSearchMailboxConfiguration {
private static final String OPENSEARCH_INDEX_HEADERS =
"opensearch.indexHeaders";
private static final String OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE =
"opensearch.message.index.optimize.move";
private static final String OPENSEARCH_INDEX_BODY = "opensearch.indexBody";
+ private static final String OPENSEARCH_INDEX_USER = "opensearch.indexUser";
private static final boolean DEFAULT_INDEX_ATTACHMENTS = true;
private static final boolean DEFAULT_INDEX_HEADERS = true;
public static final boolean DEFAULT_OPTIMIZE_MOVES = false;
public static final boolean DEFAULT_INDEX_BODY = true;
+ public static final boolean DEFAULT_INDEX_USER = false;
public static final OpenSearchMailboxConfiguration DEFAULT_CONFIGURATION =
builder().build();
public static OpenSearchMailboxConfiguration fromProperties(Configuration
configuration) {
@@ -124,6 +135,7 @@ public class OpenSearchMailboxConfiguration {
.indexHeaders(provideIndexHeaders(configuration))
.optimiseMoves(configuration.getBoolean(OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE,
null))
.indexBody(provideIndexBody(configuration))
+ .indexUser(provideIndexUser(configuration))
.build();
}
@@ -169,6 +181,13 @@ public class OpenSearchMailboxConfiguration {
return IndexBody.NO;
}
+ private static IndexUser provideIndexUser(Configuration configuration) {
+ if (configuration.getBoolean(OPENSEARCH_INDEX_USER,
DEFAULT_INDEX_USER)) {
+ return IndexUser.YES;
+ }
+ return IndexUser.NO;
+ }
+
private final IndexName indexMailboxName;
private final ReadAliasName readAliasMailboxName;
private final WriteAliasName writeAliasMailboxName;
@@ -176,10 +195,12 @@ public class OpenSearchMailboxConfiguration {
private final IndexHeaders indexHeaders;
private final boolean optimiseMoves;
private final IndexBody indexBody;
+ private final IndexUser indexUser;
private OpenSearchMailboxConfiguration(IndexName indexMailboxName,
ReadAliasName readAliasMailboxName,
WriteAliasName
writeAliasMailboxName, IndexAttachments indexAttachment,
- IndexHeaders indexHeaders, boolean
optimiseMoves, IndexBody indexBody) {
+ IndexHeaders indexHeaders, boolean
optimiseMoves, IndexBody indexBody,
+ IndexUser indexUser) {
this.indexMailboxName = indexMailboxName;
this.readAliasMailboxName = readAliasMailboxName;
this.writeAliasMailboxName = writeAliasMailboxName;
@@ -187,6 +208,7 @@ public class OpenSearchMailboxConfiguration {
this.indexHeaders = indexHeaders;
this.optimiseMoves = optimiseMoves;
this.indexBody = indexBody;
+ this.indexUser = indexUser;
}
public IndexName getIndexMailboxName() {
@@ -217,6 +239,10 @@ public class OpenSearchMailboxConfiguration {
return indexBody;
}
+ public IndexUser getIndexUser() {
+ return indexUser;
+ }
+
@Override
public final boolean equals(Object o) {
if (o instanceof OpenSearchMailboxConfiguration) {
@@ -228,7 +254,8 @@ public class OpenSearchMailboxConfiguration {
&& Objects.equals(this.readAliasMailboxName,
that.readAliasMailboxName)
&& Objects.equals(this.optimiseMoves, that.optimiseMoves)
&& Objects.equals(this.writeAliasMailboxName,
that.writeAliasMailboxName)
- && Objects.equals(this.indexBody, that.indexBody);
+ && Objects.equals(this.indexBody, that.indexBody)
+ && Objects.equals(this.indexUser, that.indexUser);
}
return false;
}
@@ -236,6 +263,6 @@ public class OpenSearchMailboxConfiguration {
@Override
public final int hashCode() {
return Objects.hash(indexMailboxName, readAliasMailboxName,
writeAliasMailboxName, indexAttachment, indexHeaders,
- writeAliasMailboxName, optimiseMoves, indexBody);
+ writeAliasMailboxName, optimiseMoves, indexBody, indexUser);
}
}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
index 1ec770400b..1dbc729adc 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
@@ -357,7 +357,7 @@ public class OpenSearchListeningMessageSearchIndex extends
ListeningMessageSearc
}
private Mono<String> generateIndexedJson(Mailbox mailbox, MailboxMessage
message, MailboxSession session) {
- return messageToOpenSearchJson.convertToJson(message)
+ return messageToOpenSearchJson.convertToJson(message, session)
.onErrorResume(e -> {
LOGGER.warn("Indexing mailbox {}-{} of user {} on message {}
without attachments ",
mailbox.getName(),
@@ -365,7 +365,7 @@ public class OpenSearchListeningMessageSearchIndex extends
ListeningMessageSearc
session.getUser().asString(),
message.getUid(),
e);
- return
messageToOpenSearchJson.convertToJsonWithoutAttachment(message);
+ return
messageToOpenSearchJson.convertToJsonWithoutAttachment(message, session);
});
}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
index f068801a5e..23569aed72 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
@@ -38,6 +38,7 @@ import
org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.search.SearchUtil;
import org.apache.james.mime4j.MimeException;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -64,6 +65,7 @@ public class IndexableMessage {
private MailboxMessage message;
private TextExtractor textExtractor;
private ZoneId zoneId;
+ private String user = null;
private Builder() {
}
@@ -112,6 +114,13 @@ public class IndexableMessage {
return this;
}
+ public Builder user(MessageToOpenSearchJson.IndexUser indexUser,
String user) {
+ if (indexUser == MessageToOpenSearchJson.IndexUser.YES) {
+ this.user = user;
+ }
+ return this;
+ }
+
private Mono<IndexableMessage> instantiateIndexedMessage() throws
IOException, MimeException {
String messageId =
SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message);
String threadId =
SearchUtil.getSerializedThreadIdIfSupportedByUnderlyingStorageOrNull(message);
@@ -175,6 +184,7 @@ public class IndexableMessage {
mediaType,
messageId,
threadId,
+ user,
modSeq,
sentDate,
saveDate,
@@ -183,8 +193,7 @@ public class IndexableMessage {
subType,
to,
uid,
- userFlags,
- mimeMessageID);
+ userFlags, mimeMessageID);
});
}
@@ -236,6 +245,7 @@ public class IndexableMessage {
private final String mediaType;
private final String messageId;
private final String threadId;
+ private final String user;
private final long modSeq;
private final String sentDate;
private final Optional<String> saveDate;
@@ -264,7 +274,7 @@ public class IndexableMessage {
boolean isUnRead,
String mailboxId,
String mediaType, String messageId,
- String threadId,
+ String threadId, String user,
ModSeq modSeq,
String sentDate,
Optional<String> saveDate, long size,
@@ -293,6 +303,7 @@ public class IndexableMessage {
this.mediaType = mediaType;
this.messageId = messageId;
this.threadId = threadId;
+ this.user = user;
this.modSeq = modSeq.asLong();
this.sentDate = sentDate;
this.saveDate = saveDate;
@@ -445,6 +456,12 @@ public class IndexableMessage {
return isUnRead;
}
+ @JsonProperty(JsonMessageConstants.USER)
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public String getUser() {
+ return user;
+ }
+
@JsonProperty(JsonMessageConstants.MIME_MESSAGE_ID)
public Optional<String> getMimeMessageID() {
return mimeMessageID;
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/JsonMessageConstants.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/JsonMessageConstants.java
index 798f4e6455..65cef55261 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/JsonMessageConstants.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/JsonMessageConstants.java
@@ -47,6 +47,7 @@ public interface JsonMessageConstants {
String ATTACHMENTS = "attachments";
String TEXT = "text";
String MIME_MESSAGE_ID = "mimeMessageID";
+ String USER = "user";
String MODSEQ = "modSeq";
String USER_FLAGS = "userFlags";
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
index fa47d7ac87..4253dfa9dd 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
@@ -28,6 +28,7 @@ import java.util.Date;
import jakarta.inject.Inject;
import jakarta.mail.Flags;
+import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.MessageUid;
import org.apache.james.mailbox.ModSeq;
import org.apache.james.mailbox.extractor.TextExtractor;
@@ -49,6 +50,9 @@ import com.google.common.base.Preconditions;
import reactor.core.publisher.Mono;
public class MessageToOpenSearchJson {
+ public enum IndexUser {
+ YES, NO
+ }
private final ObjectMapper mapper;
private final TextExtractor textExtractor;
@@ -56,12 +60,17 @@ public class MessageToOpenSearchJson {
private final IndexAttachments indexAttachments;
private final IndexHeaders indexHeaders;
private final IndexBody indexBody;
+ private final IndexUser indexUser;
public MessageToOpenSearchJson(TextExtractor textExtractor, ZoneId zoneId,
IndexAttachments indexAttachments, IndexHeaders indexHeaders) {
this(textExtractor, zoneId, indexAttachments, indexHeaders,
IndexBody.YES);
}
public MessageToOpenSearchJson(TextExtractor textExtractor, ZoneId zoneId,
IndexAttachments indexAttachments, IndexHeaders indexHeaders, IndexBody
indexBody) {
+ this(textExtractor, zoneId, indexAttachments, indexHeaders, indexBody,
IndexUser.YES);
+ }
+
+ public MessageToOpenSearchJson(TextExtractor textExtractor, ZoneId zoneId,
IndexAttachments indexAttachments, IndexHeaders indexHeaders, IndexBody
indexBody, IndexUser indexUser) {
this.textExtractor = textExtractor;
this.zoneId = zoneId;
this.indexAttachments = indexAttachments;
@@ -70,11 +79,41 @@ public class MessageToOpenSearchJson {
this.mapper.registerModule(new GuavaModule());
this.mapper.registerModule(new Jdk8Module());
this.indexBody = indexBody;
+ this.indexUser = indexUser;
}
@Inject
public MessageToOpenSearchJson(TextExtractor textExtractor,
OpenSearchMailboxConfiguration configuration) {
- this(textExtractor, ZoneId.systemDefault(),
configuration.getIndexAttachment(), configuration.getIndexHeaders(),
configuration.getIndexBody());
+ this(textExtractor, ZoneId.systemDefault(),
configuration.getIndexAttachment(), configuration.getIndexHeaders(),
+ configuration.getIndexBody(), configuration.getIndexUser());
+ }
+
+ public Mono<String> convertToJson(MailboxMessage message, MailboxSession
session) {
+ Preconditions.checkNotNull(message);
+
+ return IndexableMessage.builder()
+ .message(message)
+ .extractor(textExtractor)
+ .zoneId(zoneId)
+ .indexAttachments(indexAttachments)
+ .indexHeaders(indexHeaders)
+ .indexBody(indexBody)
+ .user(indexUser, session.getUser().asString())
+ .build()
+ .map(Throwing.function(mapper::writeValueAsString));
+ }
+
+ public Mono<String> convertToJsonWithoutAttachment(MailboxMessage message,
MailboxSession session) {
+ return IndexableMessage.builder()
+ .message(message)
+ .extractor(textExtractor)
+ .zoneId(zoneId)
+ .indexAttachments(IndexAttachments.NO)
+ .indexHeaders(indexHeaders)
+ .user(indexUser, session.getUser().asString())
+ .indexBody(indexBody)
+ .build()
+ .map(Throwing.function(mapper::writeValueAsString));
}
public Mono<String> convertToJson(MailboxMessage message) {
diff --git a/src/site/xdoc/server/config-opensearch.xml
b/src/site/xdoc/server/config-opensearch.xml
index 6dda770d3c..e6c5f95f3c 100644
--- a/src/site/xdoc/server/config-opensearch.xml
+++ b/src/site/xdoc/server/config-opensearch.xml
@@ -124,6 +124,8 @@
Default to false.</dd>
<dt><strong>opensearch.indexBody</strong></dt>
<dd>Indicates if you wish to index body or not (default: true). This
can be used to decrease the performance cost associated with indexing.</dd>
+ <dt><strong>opensearch.indexUser</strong></dt>
+ <dd>Indicates if you wish to index user or not (default: false).
This can be used to have per user reports in OpenSearch Dashboards.</dd>
<dt><strong>opensearch.index.quota.ratio.name</strong></dt>
<dd>Specify the OpenSearch alias name used for quotas</dd>
<dt><strong>opensearch.alias.read.quota.ratio.name</strong></dt>
diff --git a/upgrade-instructions.md b/upgrade-instructions.md
index d1a308f9a2..2d9bfe8b52 100644
--- a/upgrade-instructions.md
+++ b/upgrade-instructions.md
@@ -32,8 +32,9 @@ Change list:
- [Make all queues on RabbitMQ quorum queue when
`quorum.queues.enable=true`](#make-all-queues-on-rabbitmq-quorum-queue-when-quorumqueuesenabletrue)
- [Migrate RabbitMQ classic queues to version
2](#migrate-rabbitmq-classic-queues-to-version-2)
- [JAMES-3946 White list removals](#james-3946-white-list-removals)
+ - [JAMES-4052 Details in quota index](#james-4052-details-in-quota-index)
-### JAMES-4052 Details in quota index
+### JAMES-4052 Details in quota index and mailbox user
Date: 23/07/2024
@@ -80,6 +81,29 @@
opensearch.alias.read.quota.ratio.name=james-quota-ratio-read-v2
opensearch.alias.write.quota.ratio.name=james-quota-ratio-write-v2
```
+We also added an optional field to the mailbox mapping for `user`. It allows
doing per user analysis in OpenSearch dashboards.
+
+In order to activate that field, include in `opensearch.properties`:
+
+```
+opensearch.indexUser=true
+```
+
+And create the field:
+
+```
+curl -X PUT \
+ http://ip:port/mailbox_v2/_mapping \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "properties": {
+ "user": {
+ "type": "keyword"
+ }
+ }
+}'
+```
+
### JAMES-3946 White list removals
Date: 14/06/2024
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
