This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 667c27e6b9 JAMES-4010 Do not index body (#2018)
667c27e6b9 is described below
commit 667c27e6b9af0dd72a70679a6bcaee0096e3fda9
Author: Benoit TELLIER <btell...@linagora.com>
AuthorDate: Sun Feb 25 23:04:26 2024 +0100
JAMES-4010 Do not index body (#2018)
This relax pressure on the AES blob store and APPEND
Co-authored-by: Rene Cordier <rcord...@linagora.com>
---
.../apache/james/mailbox/opensearch/IndexBody.java | 24 +++
.../opensearch/OpenSearchMailboxConfiguration.java | 37 +++-
.../OpenSearchListeningMessageSearchIndex.java | 12 +-
.../mailbox/opensearch/json/IndexableMessage.java | 17 +-
.../opensearch/json/MessageToOpenSearchJson.java | 14 +-
.../mailbox/opensearch/json/MimePartParser.java | 10 +-
.../OpenSearchMailboxConfigurationTest.java | 97 ++++++----
.../OpenSearchNoIndexBodyIntegrationTest.java | 197 +++++++++++++++++++++
.../store/search/ListeningMessageSearchIndex.java | 10 +-
.../modules/ROOT/pages/configure/opensearch.adoc | 3 +
src/site/xdoc/server/config-opensearch.xml | 2 +
11 files changed, 372 insertions(+), 51 deletions(-)
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/IndexBody.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/IndexBody.java
new file mode 100644
index 0000000000..b5dd068701
--- /dev/null
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/IndexBody.java
@@ -0,0 +1,24 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.opensearch;
+
+public enum IndexBody {
+ NO, YES
+}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
index 094409581e..6fa68d0286 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
@@ -35,8 +35,8 @@ public class OpenSearchMailboxConfiguration {
private Optional<WriteAliasName> writeAliasMailboxName;
private Optional<IndexAttachments> indexAttachment;
private Optional<IndexHeaders> indexHeaders;
-
private Optional<Boolean> optimiseMoves;
+ private Optional<IndexBody> indexBody;
Builder() {
indexMailboxName = Optional.empty();
@@ -45,6 +45,7 @@ public class OpenSearchMailboxConfiguration {
indexAttachment = Optional.empty();
indexHeaders = Optional.empty();
optimiseMoves = Optional.empty();
+ indexBody = Optional.empty();
}
public Builder indexMailboxName(Optional<IndexName> indexMailboxName) {
@@ -77,6 +78,11 @@ public class OpenSearchMailboxConfiguration {
return this;
}
+ public Builder indexBody(IndexBody indexBody) {
+ this.indexBody = Optional.ofNullable(indexBody);
+ return this;
+ }
+
public OpenSearchMailboxConfiguration build() {
return new OpenSearchMailboxConfiguration(
indexMailboxName.orElse(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX),
@@ -84,7 +90,8 @@ public class OpenSearchMailboxConfiguration {
writeAliasMailboxName.orElse(MailboxOpenSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS),
indexAttachment.orElse(IndexAttachments.YES),
indexHeaders.orElse(IndexHeaders.YES),
- optimiseMoves.orElse(DEFAULT_OPTIMIZE_MOVES));
+ optimiseMoves.orElse(DEFAULT_OPTIMIZE_MOVES),
+ indexBody.orElse(IndexBody.YES));
}
}
@@ -101,9 +108,11 @@ public class OpenSearchMailboxConfiguration {
private static final String OPENSEARCH_INDEX_ATTACHMENTS =
"opensearch.indexAttachments";
private static final String OPENSEARCH_INDEX_HEADERS =
"opensearch.indexHeaders";
private static final String OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE =
"opensearch.message.index.optimize.move";
+ private static final String OPENSEARCH_INDEX_BODY = "opensearch.indexBody";
private static final boolean DEFAULT_INDEX_ATTACHMENTS = true;
private static final boolean DEFAULT_INDEX_HEADERS = true;
public static final boolean DEFAULT_OPTIMIZE_MOVES = false;
+ public static final boolean DEFAULT_INDEX_BODY = true;
public static final OpenSearchMailboxConfiguration DEFAULT_CONFIGURATION =
builder().build();
public static OpenSearchMailboxConfiguration fromProperties(Configuration
configuration) {
@@ -114,6 +123,7 @@ public class OpenSearchMailboxConfiguration {
.indexAttachment(provideIndexAttachments(configuration))
.indexHeaders(provideIndexHeaders(configuration))
.optimiseMoves(configuration.getBoolean(OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE,
null))
+ .indexBody(provideIndexBody(configuration))
.build();
}
@@ -151,25 +161,32 @@ public class OpenSearchMailboxConfiguration {
}
return IndexHeaders.NO;
}
-
+
+ private static IndexBody provideIndexBody(Configuration configuration) {
+ if (configuration.getBoolean(OPENSEARCH_INDEX_BODY,
DEFAULT_INDEX_BODY)) {
+ return IndexBody.YES;
+ }
+ return IndexBody.NO;
+ }
private final IndexName indexMailboxName;
private final ReadAliasName readAliasMailboxName;
private final WriteAliasName writeAliasMailboxName;
private final IndexAttachments indexAttachment;
private final IndexHeaders indexHeaders;
-
private final boolean optimiseMoves;
+ private final IndexBody indexBody;
private OpenSearchMailboxConfiguration(IndexName indexMailboxName,
ReadAliasName readAliasMailboxName,
WriteAliasName
writeAliasMailboxName, IndexAttachments indexAttachment,
- IndexHeaders indexHeaders, boolean
optimiseMoves) {
+ IndexHeaders indexHeaders, boolean
optimiseMoves, IndexBody indexBody) {
this.indexMailboxName = indexMailboxName;
this.readAliasMailboxName = readAliasMailboxName;
this.writeAliasMailboxName = writeAliasMailboxName;
this.indexAttachment = indexAttachment;
this.indexHeaders = indexHeaders;
this.optimiseMoves = optimiseMoves;
+ this.indexBody = indexBody;
}
public IndexName getIndexMailboxName() {
@@ -196,6 +213,10 @@ public class OpenSearchMailboxConfiguration {
return optimiseMoves;
}
+ public IndexBody getIndexBody() {
+ return indexBody;
+ }
+
@Override
public final boolean equals(Object o) {
if (o instanceof OpenSearchMailboxConfiguration) {
@@ -206,13 +227,15 @@ public class OpenSearchMailboxConfiguration {
&& Objects.equals(this.indexMailboxName, that.indexMailboxName)
&& Objects.equals(this.readAliasMailboxName,
that.readAliasMailboxName)
&& Objects.equals(this.optimiseMoves, that.optimiseMoves)
- && Objects.equals(this.writeAliasMailboxName,
that.writeAliasMailboxName);
+ && Objects.equals(this.writeAliasMailboxName,
that.writeAliasMailboxName)
+ && Objects.equals(this.indexBody, that.indexBody);
}
return false;
}
@Override
public final int hashCode() {
- return Objects.hash(indexMailboxName, readAliasMailboxName,
writeAliasMailboxName, indexAttachment, indexHeaders, writeAliasMailboxName,
optimiseMoves);
+ return Objects.hash(indexMailboxName, readAliasMailboxName,
writeAliasMailboxName, indexAttachment, indexHeaders,
+ writeAliasMailboxName, optimiseMoves, indexBody);
}
}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
index c09a55f2ba..056b533ad2 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java
@@ -59,6 +59,7 @@ import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.MessageMetaData;
import org.apache.james.mailbox.model.SearchQuery;
import org.apache.james.mailbox.model.UpdatedFlags;
+import org.apache.james.mailbox.opensearch.IndexBody;
import org.apache.james.mailbox.opensearch.MailboxOpenSearchConstants;
import org.apache.james.mailbox.opensearch.OpenSearchMailboxConfiguration;
import org.apache.james.mailbox.opensearch.json.MessageToOpenSearchJson;
@@ -228,6 +229,7 @@ public class OpenSearchListeningMessageSearchIndex extends
ListeningMessageSearc
private final Metric reIndexNotFoundMetric;
private final IndexingStrategy indexingStrategy;
+ private final IndexBody indexBody;
@Inject
public OpenSearchListeningMessageSearchIndex(MailboxSessionMapperFactory
factory,
@@ -249,6 +251,7 @@ public class OpenSearchListeningMessageSearchIndex extends
ListeningMessageSearc
} else {
this.indexingStrategy = new NaiveIndexingStrategy();
}
+ this.indexBody = configuration.getIndexBody();
this.reIndexNotFoundMetric =
metricFactory.generate("opensearch_reindex_not_found");
LOGGER.info("OpenSearchMessageSearchIndex activated with index
strategy: {}", indexingStrategy.getClass().getSimpleName());
@@ -297,7 +300,14 @@ public class OpenSearchListeningMessageSearchIndex extends
ListeningMessageSearc
private Mono<Void> processAddedEvent(MailboxSession session,
MailboxEvents.Added addedEvent, MailboxId mailboxId) {
return factory.getMailboxMapper(session)
.findMailboxById(mailboxId)
- .flatMap(mailbox -> handleAdded(session, mailbox, addedEvent));
+ .flatMap(mailbox -> handleAdded(session, mailbox, addedEvent,
chooseFetchType()));
+ }
+
+ private FetchType chooseFetchType() {
+ if (indexBody == IndexBody.YES) {
+ return FetchType.FULL;
+ }
+ return FetchType.HEADERS;
}
@Override
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
index 00a8b23164..f068801a5e 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
@@ -20,6 +20,7 @@
package org.apache.james.mailbox.opensearch.json;
import java.io.IOException;
+import java.io.InputStream;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
@@ -31,6 +32,7 @@ import org.apache.james.mailbox.ModSeq;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.mailbox.model.MessageAttachmentMetadata;
import org.apache.james.mailbox.opensearch.IndexAttachments;
+import org.apache.james.mailbox.opensearch.IndexBody;
import org.apache.james.mailbox.opensearch.IndexHeaders;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.search.SearchUtil;
@@ -58,6 +60,7 @@ public class IndexableMessage {
private IndexAttachments indexAttachments;
private IndexHeaders indexHeaders;
+ private IndexBody indexBody = IndexBody.YES;
private MailboxMessage message;
private TextExtractor textExtractor;
private ZoneId zoneId;
@@ -94,6 +97,11 @@ public class IndexableMessage {
return this;
}
+ public Builder indexBody(IndexBody indexBody) {
+ this.indexBody = indexBody;
+ return this;
+ }
+
public Builder message(MailboxMessage message) {
this.message = message;
return this;
@@ -108,7 +116,7 @@ public class IndexableMessage {
String messageId =
SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message);
String threadId =
SearchUtil.getSerializedThreadIdIfSupportedByUnderlyingStorageOrNull(message);
- return new MimePartParser(message, textExtractor).parse()
+ return new MimePartParser(textExtractor).parse(contentStream())
.asMimePart(textExtractor)
.map(parsingResult -> {
@@ -180,6 +188,13 @@ public class IndexableMessage {
});
}
+ private InputStream contentStream() throws IOException {
+ if (indexBody == IndexBody.YES) {
+ return message.getFullContent();
+ }
+ return message.getHeaderContent();
+ }
+
private List<HeaderCollection.Header>
filterHeadersIfNeeded(List<HeaderCollection.Header> headers) {
if (IndexHeaders.YES.equals(indexHeaders)) {
return headers;
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
index 27e35b5d15..82e27b41fd 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MessageToOpenSearchJson.java
@@ -34,7 +34,9 @@ import org.apache.james.mailbox.ModSeq;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.mailbox.model.MailboxId;
import org.apache.james.mailbox.opensearch.IndexAttachments;
+import org.apache.james.mailbox.opensearch.IndexBody;
import org.apache.james.mailbox.opensearch.IndexHeaders;
+import org.apache.james.mailbox.opensearch.OpenSearchMailboxConfiguration;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -54,8 +56,13 @@ public class MessageToOpenSearchJson {
private final ZoneId zoneId;
private final IndexAttachments indexAttachments;
private final IndexHeaders indexHeaders;
+ private final IndexBody indexBody;
public MessageToOpenSearchJson(TextExtractor textExtractor, ZoneId zoneId,
IndexAttachments indexAttachments, IndexHeaders indexHeaders) {
+ this(textExtractor, zoneId, indexAttachments, indexHeaders,
IndexBody.YES);
+ }
+
+ public MessageToOpenSearchJson(TextExtractor textExtractor, ZoneId zoneId,
IndexAttachments indexAttachments, IndexHeaders indexHeaders, IndexBody
indexBody) {
this.textExtractor = textExtractor;
this.zoneId = zoneId;
this.indexAttachments = indexAttachments;
@@ -63,11 +70,12 @@ public class MessageToOpenSearchJson {
this.mapper = new ObjectMapper();
this.mapper.registerModule(new GuavaModule());
this.mapper.registerModule(new Jdk8Module());
+ this.indexBody = indexBody;
}
@Inject
- public MessageToOpenSearchJson(TextExtractor textExtractor,
IndexAttachments indexAttachments, IndexHeaders indexHeaders) {
- this(textExtractor, ZoneId.systemDefault(), indexAttachments,
indexHeaders);
+ public MessageToOpenSearchJson(TextExtractor textExtractor,
OpenSearchMailboxConfiguration configuration) {
+ this(textExtractor, ZoneId.systemDefault(),
configuration.getIndexAttachment(), configuration.getIndexHeaders(),
configuration.getIndexBody());
}
public Mono<String> convertToJson(MailboxMessage message) {
@@ -79,6 +87,7 @@ public class MessageToOpenSearchJson {
.zoneId(zoneId)
.indexAttachments(indexAttachments)
.indexHeaders(indexHeaders)
+ .indexBody(indexBody)
.build()
.map(Throwing.function(mapper::writeValueAsString));
}
@@ -90,6 +99,7 @@ public class MessageToOpenSearchJson {
.zoneId(zoneId)
.indexAttachments(IndexAttachments.NO)
.indexHeaders(indexHeaders)
+ .indexBody(indexBody)
.build()
.map(Throwing.function(mapper::writeValueAsString));
}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
index 779e0ce02b..93e6ad7d99 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
@@ -20,6 +20,7 @@
package org.apache.james.mailbox.opensearch.json;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Deque;
import java.util.LinkedList;
@@ -28,7 +29,6 @@ import java.util.Optional;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.mailbox.model.ContentType.MediaType;
import org.apache.james.mailbox.model.ContentType.SubType;
-import org.apache.james.mailbox.store.mail.model.Message;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.codec.DecodeMonitor;
import org.apache.james.mime4j.field.LenientFieldParser;
@@ -46,15 +46,13 @@ public class MimePartParser {
private static final Logger LOGGER =
LoggerFactory.getLogger(MimePartParser.class);
private static final LenientFieldParser FIELD_PARSER = new
LenientFieldParser();
- private final Message message;
private final TextExtractor textExtractor;
private final MimeTokenStream stream;
private final Deque<MimePartContainerBuilder> builderStack;
private MimePart.ParsedMimePart result;
private MimePartContainerBuilder currentlyBuildMimePart;
- public MimePartParser(Message message, TextExtractor textExtractor) {
- this.message = message;
+ public MimePartParser(TextExtractor textExtractor) {
this.textExtractor = textExtractor;
this.builderStack = new LinkedList<>();
this.currentlyBuildMimePart = new RootMimePartContainerBuilder();
@@ -63,8 +61,8 @@ public class MimePartParser {
new DefaultBodyDescriptorBuilder(null, FIELD_PARSER,
DecodeMonitor.SILENT));
}
- public MimePart.ParsedMimePart parse() throws IOException, MimeException {
- stream.parse(message.getFullContent());
+ public MimePart.ParsedMimePart parse(InputStream inputStream) throws
IOException, MimeException {
+ stream.parse(inputStream);
for (EntityState state = stream.getState(); state !=
EntityState.T_END_OF_STREAM; state = stream.next()) {
processMimePart(stream, state);
}
diff --git
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfigurationTest.java
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfigurationTest.java
index 173360cbb1..4f562e7645 100644
---
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfigurationTest.java
+++
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfigurationTest.java
@@ -31,7 +31,7 @@ import nl.jqno.equalsverifier.EqualsVerifier;
class OpenSearchMailboxConfigurationTest {
@Test
- void elasticSearchMailboxConfigurationShouldRespectBeanContract() {
+ void openSearchMailboxConfigurationShouldRespectBeanContract() {
EqualsVerifier.forClass(OpenSearchMailboxConfiguration.class)
.verify();
}
@@ -43,9 +43,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.index.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexMailboxName())
+ assertThat(openSearchConfiguration.getIndexMailboxName())
.isEqualTo(new IndexName(name));
}
@@ -57,9 +57,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.index.mailbox.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexMailboxName())
+ assertThat(openSearchConfiguration.getIndexMailboxName())
.isEqualTo(new IndexName(name));
}
@@ -70,9 +70,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.index.mailbox.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexMailboxName())
+ assertThat(openSearchConfiguration.getIndexMailboxName())
.isEqualTo(new IndexName(name));
}
@@ -81,9 +81,9 @@ class OpenSearchMailboxConfigurationTest {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexMailboxName())
+ assertThat(openSearchConfiguration.getIndexMailboxName())
.isEqualTo(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX);
}
@@ -94,9 +94,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.read.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getReadAliasMailboxName())
+ assertThat(openSearchConfiguration.getReadAliasMailboxName())
.isEqualTo(new ReadAliasName(name));
}
@@ -107,9 +107,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.read.mailbox.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getReadAliasMailboxName())
+ assertThat(openSearchConfiguration.getReadAliasMailboxName())
.isEqualTo(new ReadAliasName(name));
}
@@ -121,9 +121,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.read.name", "other");
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getReadAliasMailboxName())
+ assertThat(openSearchConfiguration.getReadAliasMailboxName())
.isEqualTo(new ReadAliasName(name));
}
@@ -132,9 +132,9 @@ class OpenSearchMailboxConfigurationTest {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getReadAliasMailboxName())
+ assertThat(openSearchConfiguration.getReadAliasMailboxName())
.isEqualTo(MailboxOpenSearchConstants.DEFAULT_MAILBOX_READ_ALIAS);
}
@@ -145,9 +145,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.write.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getWriteAliasMailboxName())
+ assertThat(openSearchConfiguration.getWriteAliasMailboxName())
.isEqualTo(new WriteAliasName(name));
}
@@ -158,9 +158,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.write.mailbox.name", name);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getWriteAliasMailboxName())
+ assertThat(openSearchConfiguration.getWriteAliasMailboxName())
.isEqualTo(new WriteAliasName(name));
}
@@ -172,9 +172,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.alias.write.name", "other");
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getWriteAliasMailboxName())
+ assertThat(openSearchConfiguration.getWriteAliasMailboxName())
.isEqualTo(new WriteAliasName(name));
}
@@ -183,9 +183,9 @@ class OpenSearchMailboxConfigurationTest {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getWriteAliasMailboxName())
+ assertThat(openSearchConfiguration.getWriteAliasMailboxName())
.isEqualTo(MailboxOpenSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS);
}
@@ -195,9 +195,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.indexAttachments", true);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexAttachment())
+ assertThat(openSearchConfiguration.getIndexAttachment())
.isEqualTo(IndexAttachments.YES);
}
@@ -207,9 +207,9 @@ class OpenSearchMailboxConfigurationTest {
configuration.addProperty("opensearch.indexAttachments", false);
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexAttachment())
+ assertThat(openSearchConfiguration.getIndexAttachment())
.isEqualTo(IndexAttachments.NO);
}
@@ -218,10 +218,45 @@ class OpenSearchMailboxConfigurationTest {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.addProperty("opensearch.hosts", "127.0.0.1");
- OpenSearchMailboxConfiguration elasticSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
- assertThat(elasticSearchConfiguration.getIndexAttachment())
+ assertThat(openSearchConfiguration.getIndexAttachment())
.isEqualTo(IndexAttachments.YES);
}
+ @Test
+ void getIndexBodyShouldReturnConfiguredValueWhenTrue() {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.addProperty("opensearch.indexBody", true);
+ configuration.addProperty("opensearch.hosts", "127.0.0.1");
+
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+
+ assertThat(openSearchConfiguration.getIndexBody())
+ .isEqualTo(IndexBody.YES);
+ }
+
+ @Test
+ void getIndexBodyShouldReturnConfiguredValueWhenFalse() {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.addProperty("opensearch.indexBody", false);
+ configuration.addProperty("opensearch.hosts", "127.0.0.1");
+
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+
+ assertThat(openSearchConfiguration.getIndexBody())
+ .isEqualTo(IndexBody.NO);
+ }
+
+ @Test
+ void getIndexBodyShouldReturnDefaultValueWhenMissing() {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.addProperty("opensearch.hosts", "127.0.0.1");
+
+ OpenSearchMailboxConfiguration openSearchConfiguration =
OpenSearchMailboxConfiguration.fromProperties(configuration);
+
+ assertThat(openSearchConfiguration.getIndexBody())
+ .isEqualTo(IndexBody.YES);
+ }
+
}
\ No newline at end of file
diff --git
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
new file mode 100644
index 0000000000..e2bc9bb823
--- /dev/null
+++
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
@@ -0,0 +1,197 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.opensearch;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Durations.ONE_HUNDRED_MILLISECONDS;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
+import java.util.UUID;
+
+import org.apache.james.backends.opensearch.DockerOpenSearchExtension;
+import org.apache.james.backends.opensearch.IndexName;
+import org.apache.james.backends.opensearch.OpenSearchIndexer;
+import org.apache.james.backends.opensearch.ReactorOpenSearchClient;
+import org.apache.james.backends.opensearch.ReadAliasName;
+import org.apache.james.backends.opensearch.WriteAliasName;
+import org.apache.james.core.Username;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageManager;
+import org.apache.james.mailbox.inmemory.InMemoryMailboxManager;
+import org.apache.james.mailbox.inmemory.InMemoryMessageId;
+import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
+import org.apache.james.mailbox.model.ComposedMessageId;
+import org.apache.james.mailbox.model.Mailbox;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.mailbox.model.SearchQuery;
+import
org.apache.james.mailbox.opensearch.events.OpenSearchListeningMessageSearchIndex;
+import org.apache.james.mailbox.opensearch.json.MessageToOpenSearchJson;
+import org.apache.james.mailbox.opensearch.query.CriterionConverter;
+import org.apache.james.mailbox.opensearch.query.QueryConverter;
+import org.apache.james.mailbox.opensearch.search.OpenSearchSearcher;
+import org.apache.james.mailbox.store.StoreMessageManager;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
+import org.apache.james.mailbox.tika.TikaConfiguration;
+import org.apache.james.mailbox.tika.TikaExtension;
+import org.apache.james.mailbox.tika.TikaHttpClientImpl;
+import org.apache.james.mailbox.tika.TikaTextExtractor;
+import org.apache.james.metrics.tests.RecordingMetricFactory;
+import org.apache.james.mime4j.dom.Message;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
+import org.awaitility.core.ConditionFactory;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.opensearch.client.opensearch._types.query_dsl.Query;
+import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders;
+import org.opensearch.client.opensearch.core.SearchRequest;
+
+import com.google.common.collect.ImmutableSet;
+
+import reactor.core.publisher.Mono;
+
+public class OpenSearchNoIndexBodyIntegrationTest {
+ static final int SEARCH_SIZE = 1;
+ private static final Username USERNAME = Username.of("user");
+ private static final ConditionFactory CALMLY_AWAIT = Awaitility
+ .with().pollInterval(ONE_HUNDRED_MILLISECONDS)
+ .and().pollDelay(ONE_HUNDRED_MILLISECONDS)
+ .await();
+
+ @RegisterExtension
+ static TikaExtension tika = new TikaExtension();
+
+ @RegisterExtension
+ static DockerOpenSearchExtension openSearch = new
DockerOpenSearchExtension(DockerOpenSearchExtension.CleanupStrategy.NONE);
+
+ TikaTextExtractor textExtractor;
+ ReactorOpenSearchClient client;
+ private InMemoryMailboxManager storeMailboxManager;
+ private MessageSearchIndex messageSearchIndex;
+ private IndexName indexName;
+ protected MailboxSession session;
+ private Mailbox mailbox;
+ private MailboxPath inboxPath;
+
+
+ @BeforeEach
+ void setUp() throws Exception {
+ textExtractor = new TikaTextExtractor(new RecordingMetricFactory(),
+ new TikaHttpClientImpl(TikaConfiguration.builder()
+ .host(tika.getIp())
+ .port(tika.getPort())
+ .timeoutInMillis(tika.getTimeoutInMillis())
+ .build()));
+ client = openSearch.getDockerOpenSearch().clientProvider().get();
+
+ InMemoryMessageId.Factory messageIdFactory = new
InMemoryMessageId.Factory();
+ MailboxIdRoutingKeyFactory routingKeyFactory = new
MailboxIdRoutingKeyFactory();
+ ReadAliasName readAliasName = new
ReadAliasName(UUID.randomUUID().toString());
+ WriteAliasName writeAliasName = new
WriteAliasName(UUID.randomUUID().toString());
+ indexName = new IndexName(UUID.randomUUID().toString());
+ MailboxIndexCreationUtil.prepareClient(client, readAliasName,
writeAliasName, indexName,
+ openSearch.getDockerOpenSearch().configuration());
+
+ InMemoryIntegrationResources resources =
InMemoryIntegrationResources.builder()
+ .preProvisionnedFakeAuthenticator()
+ .fakeAuthorizator()
+ .inVmEventBus()
+ .defaultAnnotationLimits()
+ .defaultMessageParser()
+ .listeningSearchIndex(preInstanciationStage -> new
OpenSearchListeningMessageSearchIndex(
+ preInstanciationStage.getMapperFactory(),
+ ImmutableSet.of(),
+ new OpenSearchIndexer(client, writeAliasName),
+ new OpenSearchSearcher(client, new QueryConverter(new
CriterionConverter()), SEARCH_SIZE, readAliasName, routingKeyFactory),
+ new MessageToOpenSearchJson(textExtractor,
ZoneId.of("Europe/Paris"), IndexAttachments.YES, IndexHeaders.YES,
IndexBody.NO),
+ preInstanciationStage.getSessionProvider(), routingKeyFactory,
messageIdFactory,
+
OpenSearchMailboxConfiguration.builder().indexBody(IndexBody.NO).build(), new
RecordingMetricFactory()))
+ .noPreDeletionHooks()
+ .storeQuotaManager()
+ .build();
+
+ storeMailboxManager = resources.getMailboxManager();
+ messageSearchIndex = resources.getSearchIndex();
+
+ session = storeMailboxManager.createSystemSession(USERNAME);
+ inboxPath = MailboxPath.inbox(USERNAME);
+ storeMailboxManager.createMailbox(inboxPath, session);
+
+ StoreMessageManager inboxMessageManager = (StoreMessageManager)
storeMailboxManager.getMailbox(inboxPath, session);
+ mailbox = inboxMessageManager.getMailboxEntity();
+ }
+
+ @AfterEach
+ void tearDown() throws IOException {
+ client.close();
+ }
+
+ @Test
+ void searchingByBodyContentShouldNotReturnMessageWhenNoIndexBody() throws
Exception {
+ addMessage(session, inboxPath).block();
+
+ awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L);
+
+ SearchQuery searchQuery =
SearchQuery.of(SearchQuery.bodyContains("Hello"));
+
+ assertThat(messageSearchIndex.search(session, mailbox,
searchQuery).collectList().block())
+ .isEmpty();
+ }
+
+ @Test
+ void searchingAllShoulReturnMessageWhenNoIndexBody() throws Exception {
+ ComposedMessageId composedMessageId = addMessage(session,
inboxPath).block();
+
+ awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L);
+
+ SearchQuery searchQuery = SearchQuery.of(SearchQuery.all());
+
+ assertThat(messageSearchIndex.search(session, mailbox,
searchQuery).collectList().block())
+ .containsExactly(composedMessageId.getUid());
+ }
+
+
+ private Mono<ComposedMessageId> addMessage(MailboxSession session,
MailboxPath mailboxPath) throws Exception {
+ MessageManager messageManager =
storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "u...@example.com";
+ return
Mono.from(messageManager.appendMessageReactive(MessageManager.AppendCommand.from(
+ Message.Builder.of()
+ .setTo(recipient)
+ .setBody("Hello", StandardCharsets.UTF_8)),
+ session))
+ .map(MessageManager.AppendResult::getId);
+ }
+
+ private void awaitForOpenSearch(Query query, long totalHits) {
+ CALMLY_AWAIT.atMost(Durations.TEN_SECONDS)
+ .untilAsserted(() -> assertThat(client.search(
+ new SearchRequest.Builder()
+ .index(indexName.getValue())
+ .query(query)
+ .build())
+ .block()
+ .hits().total().value()).isEqualTo(totalHits));
+ }
+}
diff --git
a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java
b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java
index 5cab4b443b..8b38c21893 100644
---
a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java
+++
b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/ListeningMessageSearchIndex.java
@@ -123,16 +123,20 @@ public abstract class ListeningMessageSearchIndex
implements MessageSearchIndex,
}
protected Mono<Void> handleAdded(MailboxSession session, Mailbox mailbox,
Added added) {
+ return handleAdded(session, mailbox, added, FetchType.FULL);
+ }
+
+ protected Mono<Void> handleAdded(MailboxSession session, Mailbox mailbox,
Added added, FetchType fetchType) {
return Flux.fromIterable(MessageRange.toRanges(added.getUids()))
- .concatMap(range -> retrieveMailboxMessages(session, mailbox,
range))
+ .concatMap(range -> retrieveMailboxMessages(session, mailbox,
range, fetchType))
.publishOn(Schedulers.parallel())
.concatMap(mailboxMessage -> add(session, mailbox, mailboxMessage))
.then();
}
- private Flux<MailboxMessage> retrieveMailboxMessages(MailboxSession
session, Mailbox mailbox, MessageRange range) {
+ private Flux<MailboxMessage> retrieveMailboxMessages(MailboxSession
session, Mailbox mailbox, MessageRange range, FetchType fetchType) {
return factory.getMessageMapper(session)
- .findInMailboxReactive(mailbox, range, FetchType.FULL, UNLIMITED);
+ .findInMailboxReactive(mailbox, range, fetchType, UNLIMITED);
}
protected abstract Flux<MessageUid> doSearch(MailboxSession session,
Mailbox mailbox, SearchQuery searchQuery) throws MailboxException;
diff --git
a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/opensearch.adoc
b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/opensearch.adoc
index e4b7d32611..dc05511abb 100644
---
a/server/apps/distributed-app/docs/modules/ROOT/pages/configure/opensearch.adoc
+++
b/server/apps/distributed-app/docs/modules/ROOT/pages/configure/opensearch.adoc
@@ -96,6 +96,9 @@ turning off headers indexing result in non-strict compliance
with the IMAP / JMA
| When set to true, James will attempt to reindex from the indexed message
when moved.
If the message is not found, it will fall back to the old behavior (The
message will be indexed from the blobStore source)
Default to false.
+
+| opensearch.indexBody
+| Indicates if you wish to index body or not (default: true). This can be used
to decrease the performance cost associated with indexing.
|===
=== Quota search
diff --git a/src/site/xdoc/server/config-opensearch.xml
b/src/site/xdoc/server/config-opensearch.xml
index 5983902f1c..6dda770d3c 100644
--- a/src/site/xdoc/server/config-opensearch.xml
+++ b/src/site/xdoc/server/config-opensearch.xml
@@ -122,6 +122,8 @@
<dd>When set to true, James will attempt to reindex from the indexed
message when moved.
If the message is not found, it will fall back to the old
behavior (The message will be indexed from the blobStore source).
Default to false.</dd>
+ <dt><strong>opensearch.indexBody</strong></dt>
+ <dd>Indicates if you wish to index body or not (default: true). This
can be used to decrease the performance cost associated with indexing.</dd>
<dt><strong>opensearch.index.quota.ratio.name</strong></dt>
<dd>Specify the OpenSearch alias name used for quotas</dd>
<dt><strong>opensearch.alias.read.quota.ratio.name</strong></dt>
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org
For additional commands, e-mail: notifications-h...@james.apache.org
