This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 092c357b18b325c457f4d1529208cb013c4055a7 Author: Rene Cordier <rcord...@linagora.com> AuthorDate: Tue Jun 21 16:19:16 2022 +0700 JAMES-3771 Migrate mailbox opensearch to new java client --- .../opensearch/DockerOpenSearchExtension.java | 7 +- .../mailbox/opensearch/MailboxMappingFactory.java | 518 ++++++++++----------- .../OpenSearchListeningMessageSearchIndex.java | 61 +-- .../opensearch/query/CriterionConverter.java | 384 +++++++++++---- .../mailbox/opensearch/query/QueryConverter.java | 37 +- .../mailbox/opensearch/query/SortConverter.java | 42 +- .../opensearch/search/OpenSearchSearcher.java | 42 +- .../opensearch/OpenSearchIntegrationTest.java | 46 +- .../OpenSearchListeningMessageSearchIndexTest.java | 64 +-- .../opensearch/search/OpenSearchSearcherTest.java | 21 +- 10 files changed, 690 insertions(+), 532 deletions(-) diff --git a/backends-common/opensearch/src/test/java/org/apache/james/backends/opensearch/DockerOpenSearchExtension.java b/backends-common/opensearch/src/test/java/org/apache/james/backends/opensearch/DockerOpenSearchExtension.java index 021a60d790..aee8790003 100644 --- a/backends-common/opensearch/src/test/java/org/apache/james/backends/opensearch/DockerOpenSearchExtension.java +++ b/backends-common/opensearch/src/test/java/org/apache/james/backends/opensearch/DockerOpenSearchExtension.java @@ -35,13 +35,13 @@ public class DockerOpenSearchExtension implements AfterEachCallback, BeforeEachC interface CleanupStrategy { CleanupStrategy NONE = any -> {}; - void clean(DockerOpenSearch elasticSearch); + void clean(DockerOpenSearch openSearch); } public static class DefaultCleanupStrategy implements CleanupStrategy { @Override - public void clean(DockerOpenSearch elasticSearch) { - elasticSearch.cleanUpData(); + public void clean(DockerOpenSearch openSearch) { + openSearch.cleanUpData(); } } @@ -55,6 +55,7 @@ public class DockerOpenSearchExtension implements AfterEachCallback, BeforeEachC @Override public void clean(DockerOpenSearch openSearch) { Awaitility.await() + .ignoreExceptions() .until(() -> { openSearch.flushIndices(); ReactorOpenSearchClient client = openSearch.clientProvider().get(); diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java index d9813d738c..2964a8738f 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/MailboxMappingFactory.java @@ -19,298 +19,256 @@ package org.apache.james.mailbox.opensearch; -import static org.apache.james.backends.opensearch.IndexCreationFactory.ANALYZER; -import static org.apache.james.backends.opensearch.IndexCreationFactory.BOOLEAN; import static org.apache.james.backends.opensearch.IndexCreationFactory.CASE_INSENSITIVE; -import static org.apache.james.backends.opensearch.IndexCreationFactory.FIELDS; -import static org.apache.james.backends.opensearch.IndexCreationFactory.FORMAT; import static org.apache.james.backends.opensearch.IndexCreationFactory.KEEP_MAIL_AND_URL; import static org.apache.james.backends.opensearch.IndexCreationFactory.KEYWORD; -import static org.apache.james.backends.opensearch.IndexCreationFactory.LONG; -import static org.apache.james.backends.opensearch.IndexCreationFactory.NESTED; -import static org.apache.james.backends.opensearch.IndexCreationFactory.NORMALIZER; -import static org.apache.james.backends.opensearch.IndexCreationFactory.PROPERTIES; import static org.apache.james.backends.opensearch.IndexCreationFactory.RAW; -import static org.apache.james.backends.opensearch.IndexCreationFactory.REQUIRED; -import static org.apache.james.backends.opensearch.IndexCreationFactory.ROUTING; -import static org.apache.james.backends.opensearch.IndexCreationFactory.SEARCH_ANALYZER; -import static org.apache.james.backends.opensearch.IndexCreationFactory.TYPE; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import java.io.IOException; +import java.util.Map; -import org.apache.james.backends.opensearch.IndexCreationFactory; import org.apache.james.mailbox.opensearch.json.JsonMessageConstants; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.client.opensearch._types.mapping.BooleanProperty; +import org.opensearch.client.opensearch._types.mapping.DateProperty; +import org.opensearch.client.opensearch._types.mapping.DynamicMapping; +import org.opensearch.client.opensearch._types.mapping.KeywordProperty; +import org.opensearch.client.opensearch._types.mapping.LongNumberProperty; +import org.opensearch.client.opensearch._types.mapping.NestedProperty; +import org.opensearch.client.opensearch._types.mapping.ObjectProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.RoutingField; +import org.opensearch.client.opensearch._types.mapping.TextProperty; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; + +import com.google.common.collect.ImmutableMap; public class MailboxMappingFactory { private static final String STANDARD = "standard"; private static final String SIMPLE = "simple"; - private static final String STORE = "store"; - public static XContentBuilder getMappingContent() { - try { - return jsonBuilder() - .startObject() - - .field("dynamic", "strict") - - .startObject(ROUTING) - .field(REQUIRED, true) - .endObject() - - .startObject(PROPERTIES) - - .startObject(JsonMessageConstants.MESSAGE_ID) - .field(TYPE, KEYWORD) - .field(STORE, true) - .endObject() - - .startObject(JsonMessageConstants.THREAD_ID) - .field(TYPE, KEYWORD) - .endObject() - - .startObject(JsonMessageConstants.UID) - .field(TYPE, LONG) - .field(STORE, true) - .endObject() - - .startObject(JsonMessageConstants.MODSEQ) - .field(TYPE, LONG) - .endObject() - - .startObject(JsonMessageConstants.SIZE) - .field(TYPE, LONG) - .endObject() - - .startObject(JsonMessageConstants.IS_ANSWERED) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.IS_DELETED) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.IS_DRAFT) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.IS_FLAGGED) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.IS_RECENT) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.IS_UNREAD) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.DATE) - .field(TYPE, IndexCreationFactory.DATE) - .field(FORMAT, "uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") - .endObject() - - .startObject(JsonMessageConstants.SENT_DATE) - .field(TYPE, IndexCreationFactory.DATE) - .field(FORMAT, "uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") - .endObject() - - .startObject(JsonMessageConstants.SAVE_DATE) - .field(TYPE, IndexCreationFactory.DATE) - .field(FORMAT, "uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") - .endObject() - - .startObject(JsonMessageConstants.USER_FLAGS) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - - .startObject(JsonMessageConstants.MEDIA_TYPE) - .field(TYPE, KEYWORD) - .endObject() - - .startObject(JsonMessageConstants.SUBTYPE) - .field(TYPE, KEYWORD) - .endObject() - - .startObject(JsonMessageConstants.FROM) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.EMailer.NAME) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .endObject() - .startObject(JsonMessageConstants.EMailer.DOMAIN) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, SIMPLE) - .field(SEARCH_ANALYZER, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.EMailer.ADDRESS) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .field(SEARCH_ANALYZER, KEEP_MAIL_AND_URL) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.HEADERS) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.HEADER.NAME) - .field(TYPE, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.HEADER.VALUE) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.SUBJECT) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.TO) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.EMailer.NAME) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .endObject() - .startObject(JsonMessageConstants.EMailer.DOMAIN) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, SIMPLE) - .field(SEARCH_ANALYZER, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.EMailer.ADDRESS) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .field(SEARCH_ANALYZER, KEEP_MAIL_AND_URL) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.CC) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.EMailer.NAME) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .endObject() - .startObject(JsonMessageConstants.EMailer.DOMAIN) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, SIMPLE) - .field(SEARCH_ANALYZER, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.EMailer.ADDRESS) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .field(SEARCH_ANALYZER, KEEP_MAIL_AND_URL) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.BCC) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.EMailer.NAME) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, KEEP_MAIL_AND_URL) - .endObject() - .startObject(JsonMessageConstants.EMailer.DOMAIN) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, SIMPLE) - .field(SEARCH_ANALYZER, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.EMailer.ADDRESS) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .field(SEARCH_ANALYZER, KEEP_MAIL_AND_URL) - .startObject(FIELDS) - .startObject(RAW) - .field(TYPE, KEYWORD) - .field(NORMALIZER, CASE_INSENSITIVE) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - - .startObject(JsonMessageConstants.MAILBOX_ID) - .field(TYPE, KEYWORD) - .field(STORE, true) - .endObject() - - .startObject(JsonMessageConstants.MIME_MESSAGE_ID) - .field(TYPE, KEYWORD) - .endObject() - - .startObject(JsonMessageConstants.TEXT_BODY) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .endObject() - - .startObject(JsonMessageConstants.HTML_BODY) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .endObject() - - .startObject(JsonMessageConstants.HAS_ATTACHMENT) - .field(TYPE, BOOLEAN) - .endObject() - - .startObject(JsonMessageConstants.ATTACHMENTS) - .startObject(PROPERTIES) - .startObject(JsonMessageConstants.Attachment.FILENAME) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .endObject() - .startObject(JsonMessageConstants.Attachment.TEXT_CONTENT) - .field(TYPE, JsonMessageConstants.TEXT) - .field(ANALYZER, STANDARD) - .endObject() - .startObject(JsonMessageConstants.Attachment.MEDIA_TYPE) - .field(TYPE, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.Attachment.SUBTYPE) - .field(TYPE, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.Attachment.FILE_EXTENSION) - .field(TYPE, KEYWORD) - .endObject() - .startObject(JsonMessageConstants.Attachment.CONTENT_DISPOSITION) - .field(TYPE, KEYWORD) - .endObject() - .endObject() - .endObject() + public static TypeMapping getMappingContent() { + return new TypeMapping.Builder() + .dynamic(DynamicMapping.Strict) + .routing(new RoutingField.Builder() + .required(true) + .build()) + .properties(generateProperties()) + .build(); + } - .endObject() - .endObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + private static Map<String, Property> generateProperties() { + return new ImmutableMap.Builder<String, Property>() + .put(JsonMessageConstants.MESSAGE_ID, new Property.Builder() + .keyword(new KeywordProperty.Builder().store(true).build()) + .build()) + .put(JsonMessageConstants.THREAD_ID, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.UID, new Property.Builder() + .long_(new LongNumberProperty.Builder().store(true).build()) + .build()) + .put(JsonMessageConstants.MODSEQ, new Property.Builder() + .long_(new LongNumberProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.SIZE, new Property.Builder() + .long_(new LongNumberProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_ANSWERED, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_DELETED, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_DRAFT, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_FLAGGED, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_RECENT, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.IS_UNREAD, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.DATE, new Property.Builder() + .date(new DateProperty.Builder() + .format("uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") + .build()) + .build()) + .put(JsonMessageConstants.SENT_DATE, new Property.Builder() + .date(new DateProperty.Builder() + .format("uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") + .build()) + .build()) + .put(JsonMessageConstants.SAVE_DATE, new Property.Builder() + .date(new DateProperty.Builder() + .format("uuuu-MM-dd'T'HH:mm:ssX||uuuu-MM-dd'T'HH:mm:ssXXX||uuuu-MM-dd'T'HH:mm:ssXXXXX") + .build()) + .build()) + .put(JsonMessageConstants.USER_FLAGS, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .put(JsonMessageConstants.MEDIA_TYPE, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.SUBTYPE, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.FROM, new Property.Builder() + .object(new ObjectProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.EMailer.NAME, new Property.Builder() + .text(new TextProperty.Builder().analyzer(KEEP_MAIL_AND_URL).build()) + .build(), + JsonMessageConstants.EMailer.DOMAIN, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(SIMPLE) + .searchAnalyzer(KEYWORD) + .build()) + .build(), + JsonMessageConstants.EMailer.ADDRESS, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(STANDARD) + .searchAnalyzer(KEEP_MAIL_AND_URL) + .fields(RAW, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .build()) + .build() + )) + .build()) + .build()) + .put(JsonMessageConstants.HEADERS, new Property.Builder() + .nested(new NestedProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.HEADER.NAME, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build(), + JsonMessageConstants.HEADER.VALUE, new Property.Builder() + .text(new TextProperty.Builder().analyzer(KEEP_MAIL_AND_URL).build()) + .build() + )) + .build()) + .build()) + .put(JsonMessageConstants.SUBJECT, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(KEEP_MAIL_AND_URL) + .fields(RAW, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .build()) + .build()) + .put(JsonMessageConstants.TO, new Property.Builder() + .object(new ObjectProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.EMailer.NAME, new Property.Builder() + .text(new TextProperty.Builder().analyzer(KEEP_MAIL_AND_URL).build()) + .build(), + JsonMessageConstants.EMailer.DOMAIN, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(SIMPLE) + .searchAnalyzer(KEYWORD) + .build()) + .build(), + JsonMessageConstants.EMailer.ADDRESS, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(STANDARD) + .searchAnalyzer(KEEP_MAIL_AND_URL) + .fields(RAW, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .build()) + .build() + )) + .build()) + .build()) + .put(JsonMessageConstants.CC, new Property.Builder() + .object(new ObjectProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.EMailer.NAME, new Property.Builder() + .text(new TextProperty.Builder().analyzer(KEEP_MAIL_AND_URL).build()) + .build(), + JsonMessageConstants.EMailer.DOMAIN, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(SIMPLE) + .searchAnalyzer(KEYWORD) + .build()) + .build(), + JsonMessageConstants.EMailer.ADDRESS, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(STANDARD) + .searchAnalyzer(KEEP_MAIL_AND_URL) + .fields(RAW, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .build()) + .build() + )) + .build()) + .build()) + .put(JsonMessageConstants.BCC, new Property.Builder() + .object(new ObjectProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.EMailer.NAME, new Property.Builder() + .text(new TextProperty.Builder().analyzer(KEEP_MAIL_AND_URL).build()) + .build(), + JsonMessageConstants.EMailer.DOMAIN, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(SIMPLE) + .searchAnalyzer(KEYWORD) + .build()) + .build(), + JsonMessageConstants.EMailer.ADDRESS, new Property.Builder() + .text(new TextProperty.Builder() + .analyzer(STANDARD) + .searchAnalyzer(KEEP_MAIL_AND_URL) + .fields(RAW, new Property.Builder() + .keyword(new KeywordProperty.Builder().normalizer(CASE_INSENSITIVE).build()) + .build()) + .build()) + .build() + )) + .build()) + .build()) + .put(JsonMessageConstants.MAILBOX_ID, new Property.Builder() + .keyword(new KeywordProperty.Builder().store(true).build()) + .build()) + .put(JsonMessageConstants.MIME_MESSAGE_ID, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.TEXT_BODY, new Property.Builder() + .text(new TextProperty.Builder().analyzer(STANDARD).build()) + .build()) + .put(JsonMessageConstants.HTML_BODY, new Property.Builder() + .text(new TextProperty.Builder().analyzer(STANDARD).build()) + .build()) + .put(JsonMessageConstants.HAS_ATTACHMENT, new Property.Builder() + .boolean_(new BooleanProperty.Builder().build()) + .build()) + .put(JsonMessageConstants.ATTACHMENTS, new Property.Builder() + .object(new ObjectProperty.Builder() + .properties(ImmutableMap.of( + JsonMessageConstants.Attachment.FILENAME, new Property.Builder() + .text(new TextProperty.Builder().analyzer(STANDARD).build()) + .build(), + JsonMessageConstants.Attachment.TEXT_CONTENT, new Property.Builder() + .text(new TextProperty.Builder().analyzer(STANDARD).build()) + .build(), + JsonMessageConstants.Attachment.MEDIA_TYPE, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build(), + JsonMessageConstants.Attachment.SUBTYPE, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build(), + JsonMessageConstants.Attachment.FILE_EXTENSION, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build(), + JsonMessageConstants.Attachment.CONTENT_DISPOSITION, new Property.Builder() + .keyword(new KeywordProperty.Builder().build()) + .build() + )) + .build()) + .build()) + .build(); } } diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java index 6b78c920d3..420d734d31 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndex.java @@ -28,12 +28,10 @@ import static org.apache.james.mailbox.opensearch.json.JsonMessageConstants.IS_U import static org.apache.james.mailbox.opensearch.json.JsonMessageConstants.MAILBOX_ID; import static org.apache.james.mailbox.opensearch.json.JsonMessageConstants.MESSAGE_ID; import static org.apache.james.mailbox.opensearch.json.JsonMessageConstants.UID; -import static org.opensearch.index.query.QueryBuilders.termQuery; import java.util.Collection; import java.util.EnumSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -63,14 +61,20 @@ import org.apache.james.mailbox.opensearch.search.OpenSearchSearcher; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; -import org.opensearch.action.get.GetResponse; -import org.opensearch.common.document.DocumentField; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.search.SearchHit; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch.core.GetResponse; +import org.opensearch.client.opensearch.core.search.Hit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.fge.lambdas.Throwing; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -193,12 +197,12 @@ public class OpenSearchListeningMessageSearchIndex extends ListeningMessageSearc @Override public Mono<Void> deleteAll(MailboxSession session, MailboxId mailboxId) { - TermQueryBuilder queryBuilder = termQuery( - MAILBOX_ID, - mailboxId.serialize()); + Query query = TermQuery.of(t -> t + .field(MAILBOX_ID) + .value(new FieldValue.Builder().stringValue(mailboxId.serialize()).build()))._toQuery(); return openSearchIndexer - .deleteAllMatchingQuery(queryBuilder, routingKeyFactory.from(mailboxId)); + .deleteAllMatchingQuery(query, routingKeyFactory.from(mailboxId)); } @Override @@ -237,12 +241,12 @@ public class OpenSearchListeningMessageSearchIndex extends ListeningMessageSearc RoutingKey routingKey = routingKeyFactory.from(mailbox.getMailboxId()); return openSearchIndexer.get(indexIdFor(mailbox.getMailboxId(), uid), routingKey) - .filter(GetResponse::isExists) - .map(GetResponse::getSourceAsMap) + .filter(GetResponse::found) + .map(GetResponse::source) .map(this::extractFlags); } - private Flags extractFlags(Map<String, Object> source) { + private Flags extractFlags(ObjectNode source) { FlagsBuilder flagsBuilder = FlagsBuilder.builder() .isAnswered(extractFlag(source, IS_ANSWERED)) .isDeleted(extractFlag(source, IS_DELETED)) @@ -251,37 +255,38 @@ public class OpenSearchListeningMessageSearchIndex extends ListeningMessageSearc .isRecent(extractFlag(source, IS_RECENT)) .isSeen(!extractFlag(source, IS_UNREAD)); - for (String userFlag : extractUserFlags(source)) { - flagsBuilder.add(userFlag); + for (JsonNode userFlag : extractUserFlags(source)) { + flagsBuilder.add(userFlag.textValue()); } return flagsBuilder.build(); } - private boolean extractFlag(Map<String, Object> source, String flag) { - return (Boolean) source.get(flag); + private boolean extractFlag(ObjectNode source, String flag) { + return source.get(flag).asBoolean(); } - private List<String> extractUserFlags(Map<String, Object> source) { - return (List<String>) source.get("userFlags"); + private ArrayNode extractUserFlags(ObjectNode source) { + return source.withArray("userFlags"); } - private void extractMessageIdFromHit(SearchHit hit, SynchronousSink<MessageId> sink) { - DocumentField messageId = hit.field(MESSAGE_ID); + private void extractMessageIdFromHit(Hit<ObjectNode> hit, SynchronousSink<MessageId> sink) { + JsonData messageId = hit.fields().get(MESSAGE_ID); if (messageId != null) { - sink.next(messageIdFactory.fromString(messageId.getValue())); + List<String> extractMessageId = messageId.deserialize(JsonpDeserializer.arrayDeserializer(JsonpDeserializer.stringDeserializer())); + sink.next(messageIdFactory.fromString(extractMessageId.get(0))); } else { - LOGGER.warn("Can not extract UID, MessageID and/or MailboxId for search result {}", hit.getId()); + LOGGER.warn("Can not extract UID, MessageID and/or MailboxId for search result {}", hit.id()); } } - private void extractUidFromHit(SearchHit hit, SynchronousSink<MessageUid> sink) { - DocumentField uid = hit.field(UID); + private void extractUidFromHit(Hit<ObjectNode> hit, SynchronousSink<MessageUid> sink) { + JsonData uid = hit.fields().get(UID); if (uid != null) { - Number uidAsNumber = uid.getValue(); - sink.next(MessageUid.of(uidAsNumber.longValue())); + List<Number> uidAsNumber = uid.deserialize(JsonpDeserializer.arrayDeserializer(JsonpDeserializer.numberDeserializer())); + sink.next(MessageUid.of(uidAsNumber.get(0).longValue())); } else { - LOGGER.warn("Can not extract UID, MessageID and/or MailboxId for search result {}", hit.getId()); + LOGGER.warn("Can not extract UID, MessageID and/or MailboxId for search result {}", hit.id()); } } } diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java index 7513fda265..d7b019c8ce 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java @@ -20,12 +20,6 @@ package org.apache.james.mailbox.opensearch.query; import static org.apache.james.backends.opensearch.IndexCreationFactory.RAW; -import static org.opensearch.index.query.QueryBuilders.boolQuery; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.index.query.QueryBuilders.matchQuery; -import static org.opensearch.index.query.QueryBuilders.nestedQuery; -import static org.opensearch.index.query.QueryBuilders.rangeQuery; -import static org.opensearch.index.query.QueryBuilders.termQuery; import java.time.format.DateTimeFormatter; import java.util.Arrays; @@ -34,7 +28,6 @@ import java.util.Locale; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collector; import java.util.stream.Stream; import javax.mail.Flags; @@ -44,15 +37,21 @@ import org.apache.james.mailbox.model.SearchQuery.Criterion; import org.apache.james.mailbox.model.SearchQuery.HeaderOperator; import org.apache.james.mailbox.opensearch.json.HeaderCollection; import org.apache.james.mailbox.opensearch.json.JsonMessageConstants; -import org.apache.lucene.search.join.ScoreMode; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.client.opensearch._types.query_dsl.MatchAllQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.NestedQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.RangeQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; public class CriterionConverter { - private final Map<Class<?>, Function<Criterion, QueryBuilder>> criterionConverterMap; - private final Map<Class<?>, BiFunction<String, HeaderOperator, QueryBuilder>> headerOperatorConverterMap; + private final Map<Class<?>, Function<Criterion, Query>> criterionConverterMap; + private final Map<Class<?>, BiFunction<String, HeaderOperator, Query>> headerOperatorConverterMap; public CriterionConverter() { criterionConverterMap = new HashMap<>(); @@ -72,7 +71,7 @@ public class CriterionConverter { registerCriterionConverter(SearchQuery.CustomFlagCriterion.class, this::convertCustomFlagCriterion); registerCriterionConverter(SearchQuery.AllCriterion.class, - criterion -> matchAllQuery()); + criterion -> new MatchAllQuery.Builder().build()._toQuery()); registerCriterionConverter(SearchQuery.ModSeqCriterion.class, criterion -> createNumericFilter(JsonMessageConstants.MODSEQ, criterion.getOperator())); @@ -92,17 +91,23 @@ public class CriterionConverter { } @SuppressWarnings("unchecked") - private <T extends Criterion> void registerCriterionConverter(Class<T> type, Function<T, QueryBuilder> f) { - criterionConverterMap.put(type, (Function<Criterion, QueryBuilder>) f); + private <T extends Criterion> void registerCriterionConverter(Class<T> type, Function<T, Query> f) { + criterionConverterMap.put(type, (Function<Criterion, Query>) f); } private void registerHeaderOperatorConverters() { registerHeaderOperatorConverter( SearchQuery.ExistsOperator.class, - (headerName, operator) -> - nestedQuery(JsonMessageConstants.HEADERS, - termQuery(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.NAME, headerName), - ScoreMode.Avg)); + (headerName, operator) -> new NestedQuery.Builder() + .path(JsonMessageConstants.HEADERS) + .query(new TermQuery.Builder() + .field(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.NAME) + .value(new FieldValue.Builder().stringValue(headerName).build()) + .build() + ._toQuery()) + .scoreMode(ChildScoreMode.Avg) + .build() + ._toQuery()); registerHeaderOperatorConverter( SearchQuery.AddressOperator.class, @@ -114,71 +119,136 @@ public class CriterionConverter { registerHeaderOperatorConverter( SearchQuery.ContainsOperator.class, - (headerName, operator) -> - nestedQuery(JsonMessageConstants.HEADERS, - boolQuery() - .must(termQuery(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.NAME, headerName)) - .must(matchQuery(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.VALUE, operator.getValue())), - ScoreMode.Avg)); + (headerName, operator) -> new NestedQuery.Builder() + .path(JsonMessageConstants.HEADERS) + .query(new BoolQuery.Builder() + .must(new TermQuery.Builder() + .field(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.NAME) + .value(new FieldValue.Builder().stringValue(headerName).build()) + .build() + ._toQuery()) + .must(new MatchQuery.Builder() + .field(JsonMessageConstants.HEADERS + "." + JsonMessageConstants.HEADER.VALUE) + .query(new FieldValue.Builder().stringValue(operator.getValue()).build()) + .build() + ._toQuery()) + .build() + ._toQuery()) + .scoreMode(ChildScoreMode.Avg) + .build() + ._toQuery()); } @SuppressWarnings("unchecked") - private <T extends HeaderOperator> void registerHeaderOperatorConverter(Class<T> type, BiFunction<String, T, QueryBuilder> f) { - headerOperatorConverterMap.put(type, (BiFunction<String, HeaderOperator, QueryBuilder>) f); + private <T extends HeaderOperator> void registerHeaderOperatorConverter(Class<T> type, BiFunction<String, T, Query> f) { + headerOperatorConverterMap.put(type, (BiFunction<String, HeaderOperator, Query>) f); } - public QueryBuilder convertCriterion(Criterion criterion) { + public Query convertCriterion(Criterion criterion) { return criterionConverterMap.get(criterion.getClass()).apply(criterion); } - private QueryBuilder convertAttachmentCriterion(SearchQuery.AttachmentCriterion criterion) { - return termQuery(JsonMessageConstants.HAS_ATTACHMENT, criterion.getOperator().isSet()); + private Query convertAttachmentCriterion(SearchQuery.AttachmentCriterion criterion) { + return new TermQuery.Builder() + .field(JsonMessageConstants.HAS_ATTACHMENT) + .value(new FieldValue.Builder().booleanValue(criterion.getOperator().isSet()).build()) + .build() + ._toQuery(); } - private QueryBuilder convertMimeMessageIDCriterion(SearchQuery.MimeMessageIDCriterion criterion) { - return termQuery(JsonMessageConstants.MIME_MESSAGE_ID, criterion.getMessageID()); + private Query convertMimeMessageIDCriterion(SearchQuery.MimeMessageIDCriterion criterion) { + return new TermQuery.Builder() + .field(JsonMessageConstants.MIME_MESSAGE_ID) + .value(new FieldValue.Builder().stringValue(criterion.getMessageID()).build()) + .build() + ._toQuery(); } - private QueryBuilder convertThreadIdCriterion(SearchQuery.ThreadIdCriterion criterion) { - return termQuery(JsonMessageConstants.THREAD_ID, criterion.getThreadId().serialize()); + private Query convertThreadIdCriterion(SearchQuery.ThreadIdCriterion criterion) { + return new TermQuery.Builder() + .field(JsonMessageConstants.THREAD_ID) + .value(new FieldValue.Builder().stringValue(criterion.getThreadId().serialize()).build()) + .build() + ._toQuery(); } - private QueryBuilder convertCustomFlagCriterion(SearchQuery.CustomFlagCriterion criterion) { - QueryBuilder termQueryBuilder = termQuery(JsonMessageConstants.USER_FLAGS, criterion.getFlag()); + private Query convertCustomFlagCriterion(SearchQuery.CustomFlagCriterion criterion) { + Query termQuery = new TermQuery.Builder() + .field(JsonMessageConstants.USER_FLAGS) + .value(new FieldValue.Builder().stringValue(criterion.getFlag()).build()) + .build() + ._toQuery(); if (criterion.getOperator().isSet()) { - return termQueryBuilder; + return termQuery; } else { - return boolQuery().mustNot(termQueryBuilder); + return new BoolQuery.Builder() + .mustNot(termQuery) + .build() + ._toQuery(); } } - private QueryBuilder convertTextCriterion(SearchQuery.TextCriterion textCriterion) { + private Query convertTextCriterion(SearchQuery.TextCriterion textCriterion) { switch (textCriterion.getType()) { case BODY: - return boolQuery() - .should(matchQuery(JsonMessageConstants.TEXT_BODY, textCriterion.getOperator().getValue())) - .should(matchQuery(JsonMessageConstants.HTML_BODY, textCriterion.getOperator().getValue())); + return new BoolQuery.Builder() + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.TEXT_BODY) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.HTML_BODY) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); case FULL: - return boolQuery() - .should(matchQuery(JsonMessageConstants.TEXT_BODY, textCriterion.getOperator().getValue())) - .should(matchQuery(JsonMessageConstants.HTML_BODY, textCriterion.getOperator().getValue())) - .should(matchQuery(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.TEXT_CONTENT, - textCriterion.getOperator().getValue())); + return new BoolQuery.Builder() + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.TEXT_BODY) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.HTML_BODY) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.TEXT_CONTENT) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); case ATTACHMENTS: - return boolQuery() - .should(matchQuery(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.TEXT_CONTENT, - textCriterion.getOperator().getValue())); + return new BoolQuery.Builder() + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.TEXT_CONTENT) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); case ATTACHMENT_FILE_NAME: - return boolQuery() - .should(termQuery(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.FILENAME, - textCriterion.getOperator().getValue())); + return new BoolQuery.Builder() + .should(new MatchQuery.Builder() + .field(JsonMessageConstants.ATTACHMENTS + "." + JsonMessageConstants.Attachment.FILENAME) + .query(new FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); + default: + throw new RuntimeException("Unknown SCOPE for text criterion"); } - throw new RuntimeException("Unknown SCOPE for text criterion"); } - private QueryBuilder dateRangeFilter(String field, SearchQuery.DateOperator dateOperator) { - return boolQuery().filter( - convertDateOperator(field, + private Query dateRangeFilter(String field, SearchQuery.DateOperator dateOperator) { + return new BoolQuery.Builder() + .filter(convertDateOperator(field, dateOperator.getType(), DateTimeFormatter.ISO_OFFSET_DATE_TIME.format( DateResolutionFormatter.computeLowerDate( @@ -187,104 +257,197 @@ public class CriterionConverter { DateTimeFormatter.ISO_OFFSET_DATE_TIME.format( DateResolutionFormatter.computeUpperDate( DateResolutionFormatter.convertDateToZonedDateTime(dateOperator.getDate()), - dateOperator.getDateResultion())))); + dateOperator.getDateResultion())))) + .build() + ._toQuery(); } - private BoolQueryBuilder convertConjunction(SearchQuery.ConjunctionCriterion criterion) { + private Query convertConjunction(SearchQuery.ConjunctionCriterion criterion) { return convertToBoolQuery(criterion.getCriteria().stream().map(this::convertCriterion), convertConjunctionType(criterion.getType())); } - private BiFunction<BoolQueryBuilder, QueryBuilder, BoolQueryBuilder> convertConjunctionType(SearchQuery.Conjunction type) { + private BiFunction<BoolQuery.Builder, Query, BoolQuery.Builder> convertConjunctionType(SearchQuery.Conjunction type) { switch (type) { case AND: - return BoolQueryBuilder::must; + return BoolQuery.Builder::must; case OR: - return BoolQueryBuilder::should; + return BoolQuery.Builder::should; case NOR: - return BoolQueryBuilder::mustNot; + return BoolQuery.Builder::mustNot; default: throw new RuntimeException("Unexpected conjunction criteria " + type); } } @SuppressWarnings("ReturnValueIgnored") - private BoolQueryBuilder convertToBoolQuery(Stream<QueryBuilder> stream, BiFunction<BoolQueryBuilder, QueryBuilder, BoolQueryBuilder> addCriterionToBoolQuery) { - return stream.collect(Collector.of(QueryBuilders::boolQuery, - addCriterionToBoolQuery::apply, - addCriterionToBoolQuery::apply)); + private Query convertToBoolQuery(Stream<Query> stream, BiFunction<BoolQuery.Builder, Query, BoolQuery.Builder> addCriterionToBoolQuery) { + BoolQuery.Builder builder = new BoolQuery.Builder(); + stream.forEach(query -> addCriterionToBoolQuery.apply(builder, query)); + return builder.build()._toQuery(); } - private QueryBuilder convertFlag(SearchQuery.FlagCriterion flagCriterion) { + private Query convertFlag(SearchQuery.FlagCriterion flagCriterion) { SearchQuery.BooleanOperator operator = flagCriterion.getOperator(); Flags.Flag flag = flagCriterion.getFlag(); if (flag.equals(Flags.Flag.DELETED)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_DELETED, operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_DELETED) + .value(new FieldValue.Builder().booleanValue(operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } if (flag.equals(Flags.Flag.ANSWERED)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_ANSWERED, operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_ANSWERED) + .value(new FieldValue.Builder().booleanValue(operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } if (flag.equals(Flags.Flag.DRAFT)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_DRAFT, operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_DRAFT) + .value(new FieldValue.Builder().booleanValue(operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } if (flag.equals(Flags.Flag.SEEN)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_UNREAD, !operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_UNREAD) + .value(new FieldValue.Builder().booleanValue(!operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } if (flag.equals(Flags.Flag.RECENT)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_RECENT, operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_RECENT) + .value(new FieldValue.Builder().booleanValue(operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } if (flag.equals(Flags.Flag.FLAGGED)) { - return boolQuery().filter(termQuery(JsonMessageConstants.IS_FLAGGED, operator.isSet())); + return new BoolQuery.Builder() + .filter(new TermQuery.Builder() + .field(JsonMessageConstants.IS_FLAGGED) + .value(new FieldValue.Builder().booleanValue(operator.isSet()).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } throw new RuntimeException("Unknown flag used in Flag search criterion"); } - private QueryBuilder createNumericFilter(String fieldName, SearchQuery.NumericOperator operator) { + private Query createNumericFilter(String fieldName, SearchQuery.NumericOperator operator) { switch (operator.getType()) { case EQUALS: - return boolQuery().filter(rangeQuery(fieldName).gte(operator.getValue()).lte(operator.getValue())); + return new BoolQuery.Builder() + .filter(new RangeQuery.Builder() + .field(fieldName) + .gte(JsonData.of(operator.getValue())) + .lte(JsonData.of(operator.getValue())) + .build() + ._toQuery()) + .build() + ._toQuery(); case GREATER_THAN: - return boolQuery().filter(rangeQuery(fieldName).gt(operator.getValue())); + return new BoolQuery.Builder() + .filter(new RangeQuery.Builder() + .field(fieldName) + .gt(JsonData.of(operator.getValue())) + .build() + ._toQuery()) + .build() + ._toQuery(); case LESS_THAN: - return boolQuery().filter(rangeQuery(fieldName).lt(operator.getValue())); + return new BoolQuery.Builder() + .filter(new RangeQuery.Builder() + .field(fieldName) + .lt(JsonData.of(operator.getValue())) + .build() + ._toQuery()) + .build() + ._toQuery(); default: throw new RuntimeException("A non existing numeric operator was triggered"); } } - private BoolQueryBuilder convertUid(SearchQuery.UidCriterion uidCriterion) { + private Query convertUid(SearchQuery.UidCriterion uidCriterion) { if (uidCriterion.getOperator().getRange().length == 0) { - return boolQuery(); + return new BoolQuery.Builder().build()._toQuery(); } - return boolQuery().filter( - convertToBoolQuery( + return new BoolQuery.Builder() + .filter(convertToBoolQuery( Arrays.stream(uidCriterion.getOperator().getRange()) - .map(this::uidRangeFilter), BoolQueryBuilder::should)); + .map(this::uidRangeFilter), BoolQuery.Builder::should)) + .build() + ._toQuery(); } - private QueryBuilder convertMessageId(SearchQuery.MessageIdCriterion messageIdCriterion) { - return termQuery(JsonMessageConstants.MESSAGE_ID, messageIdCriterion.getMessageId().serialize()); + private Query convertMessageId(SearchQuery.MessageIdCriterion messageIdCriterion) { + return new TermQuery.Builder() + .field(JsonMessageConstants.MESSAGE_ID) + .value(new FieldValue.Builder().stringValue(messageIdCriterion.getMessageId().serialize()).build()) + .build() + ._toQuery(); } - private QueryBuilder uidRangeFilter(SearchQuery.UidRange numericRange) { - return rangeQuery(JsonMessageConstants.UID) - .lte(numericRange.getHighValue().asLong()) - .gte(numericRange.getLowValue().asLong()); + private Query uidRangeFilter(SearchQuery.UidRange numericRange) { + return new RangeQuery.Builder() + .field(JsonMessageConstants.UID) + .lte(JsonData.of(numericRange.getHighValue().asLong())) + .gte(JsonData.of(numericRange.getLowValue().asLong())) + .build() + ._toQuery(); } - private QueryBuilder convertHeader(SearchQuery.HeaderCriterion headerCriterion) { + private Query convertHeader(SearchQuery.HeaderCriterion headerCriterion) { return headerOperatorConverterMap.get(headerCriterion.getOperator().getClass()) .apply( headerCriterion.getHeaderName().toLowerCase(Locale.US), headerCriterion.getOperator()); } - private QueryBuilder manageAddressFields(String headerName, String value) { - return boolQuery() - .should(matchQuery(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.NAME, value)) - .should(matchQuery(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.ADDRESS, value)) - .should(matchQuery(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.DOMAIN, value)) - .should(matchQuery(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.ADDRESS + "." + RAW, value)); + private Query manageAddressFields(String headerName, String value) { + return new BoolQuery.Builder() + .should(new MatchQuery.Builder() + .field(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.NAME) + .query(new FieldValue.Builder().stringValue(value).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.ADDRESS) + .query(new FieldValue.Builder().stringValue(value).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.DOMAIN) + .query(new FieldValue.Builder().stringValue(value).build()) + .build() + ._toQuery()) + .should(new MatchQuery.Builder() + .field(getFieldNameFromHeaderName(headerName) + "." + JsonMessageConstants.EMailer.ADDRESS + "." + RAW) + .query(new FieldValue.Builder().stringValue(value).build()) + .build() + ._toQuery()) + .build() + ._toQuery(); } private String getFieldNameFromHeaderName(String headerName) { @@ -297,20 +460,35 @@ public class CriterionConverter { return JsonMessageConstants.BCC; case HeaderCollection.FROM: return JsonMessageConstants.FROM; + default: + throw new RuntimeException("Header not recognized as Addess Header : " + headerName); } - throw new RuntimeException("Header not recognized as Addess Header : " + headerName); } - private QueryBuilder convertDateOperator(String field, SearchQuery.DateComparator dateComparator, String lowDateString, String upDateString) { + private Query convertDateOperator(String field, SearchQuery.DateComparator dateComparator, String lowDateString, String upDateString) { switch (dateComparator) { case BEFORE: - return rangeQuery(field).lt(lowDateString); // less than start of the current day + return new RangeQuery.Builder() + .field(field) + .lt(JsonData.of(upDateString)) + .build() + ._toQuery(); case AFTER: - return rangeQuery(field).gte(upDateString); // start of next day + greater than that + return new RangeQuery.Builder() + .field(field) + .gte(JsonData.of(lowDateString)) + .build() + ._toQuery(); case ON: - return rangeQuery(field).lt(upDateString).gte(lowDateString); + return new RangeQuery.Builder() + .field(field) + .lt(JsonData.of(upDateString)) + .gte(JsonData.of(lowDateString)) + .build() + ._toQuery(); + default: + throw new RuntimeException("Unknown date operator"); } - throw new RuntimeException("Unknown date operator"); } } diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/QueryConverter.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/QueryConverter.java index 1f6a34cf7e..e2d451a469 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/QueryConverter.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/QueryConverter.java @@ -19,9 +19,6 @@ package org.apache.james.mailbox.opensearch.query; -import static org.opensearch.index.query.QueryBuilders.boolQuery; -import static org.opensearch.index.query.QueryBuilders.termsQuery; - import java.util.Collection; import java.util.List; import java.util.Optional; @@ -31,14 +28,15 @@ import javax.inject.Inject; import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.mailbox.opensearch.json.JsonMessageConstants; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; import com.google.common.collect.ImmutableList; public class QueryConverter { - - private final CriterionConverter criterionConverter; @Inject @@ -46,15 +44,15 @@ public class QueryConverter { this.criterionConverter = criterionConverter; } - public QueryBuilder from(Collection<MailboxId> mailboxIds, SearchQuery query) { - BoolQueryBuilder boolQueryBuilder = boolQuery() + public Query from(Collection<MailboxId> mailboxIds, SearchQuery query) { + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder() .must(generateQueryBuilder(query)); mailboxesQuery(mailboxIds).map(boolQueryBuilder::filter); - return boolQueryBuilder; + return boolQueryBuilder.build()._toQuery(); } - private QueryBuilder generateQueryBuilder(SearchQuery searchQuery) { + private Query generateQueryBuilder(SearchQuery searchQuery) { List<SearchQuery.Criterion> criteria = searchQuery.getCriteria(); if (criteria.isEmpty()) { return criterionConverter.convertCriterion(SearchQuery.all()); @@ -65,14 +63,21 @@ public class QueryConverter { } } - private Optional<QueryBuilder> mailboxesQuery(Collection<MailboxId> mailboxIds) { + private Optional<Query> mailboxesQuery(Collection<MailboxId> mailboxIds) { if (mailboxIds.isEmpty()) { return Optional.empty(); } - ImmutableList<String> ids = mailboxIds.stream() - .map(MailboxId::serialize) - .collect(ImmutableList.toImmutableList()); - return Optional.of(termsQuery(JsonMessageConstants.MAILBOX_ID, ids)); + ImmutableList<FieldValue> ids = mailboxIds.stream() + .map(MailboxId::serialize) + .map(id -> new FieldValue.Builder().stringValue(id).build()) + .collect(ImmutableList.toImmutableList()); + return Optional.of(new TermsQuery.Builder() + .field(JsonMessageConstants.MAILBOX_ID) + .terms(new TermsQueryField.Builder() + .value(ids) + .build()) + .build() + ._toQuery()); } } diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/SortConverter.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/SortConverter.java index e1216a951f..cf4bb8d91e 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/SortConverter.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/SortConverter.java @@ -22,44 +22,44 @@ package org.apache.james.mailbox.opensearch.query; import org.apache.james.backends.opensearch.IndexCreationFactory; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.mailbox.opensearch.json.JsonMessageConstants; -import org.opensearch.search.sort.FieldSortBuilder; -import org.opensearch.search.sort.SortBuilders; -import org.opensearch.search.sort.SortMode; -import org.opensearch.search.sort.SortOrder; +import org.opensearch.client.opensearch._types.FieldSort; +import org.opensearch.client.opensearch._types.SortMode; +import org.opensearch.client.opensearch._types.SortOrder; public class SortConverter { private static final String PATH_SEPARATOR = "."; - public static FieldSortBuilder convertSort(SearchQuery.Sort sort) { + public static FieldSort convertSort(SearchQuery.Sort sort) { return getSortClause(sort.getSortClause()) .order(getOrder(sort)) - .sortMode(SortMode.MIN); + .mode(SortMode.Min) + .build(); } - private static FieldSortBuilder getSortClause(SearchQuery.Sort.SortClause clause) { + private static FieldSort.Builder getSortClause(SearchQuery.Sort.SortClause clause) { switch (clause) { case Arrival : - return SortBuilders.fieldSort(JsonMessageConstants.DATE); + return new FieldSort.Builder().field(JsonMessageConstants.DATE); case MailboxCc : - return SortBuilders.fieldSort(JsonMessageConstants.CC + PATH_SEPARATOR + JsonMessageConstants.EMailer.ADDRESS - + PATH_SEPARATOR + IndexCreationFactory.RAW); + return new FieldSort.Builder().field(JsonMessageConstants.CC + PATH_SEPARATOR + + JsonMessageConstants.EMailer.ADDRESS + PATH_SEPARATOR + IndexCreationFactory.RAW); case MailboxFrom : - return SortBuilders.fieldSort(JsonMessageConstants.FROM + PATH_SEPARATOR + JsonMessageConstants.EMailer.ADDRESS - + PATH_SEPARATOR + IndexCreationFactory.RAW); + return new FieldSort.Builder().field(JsonMessageConstants.FROM + PATH_SEPARATOR + + JsonMessageConstants.EMailer.ADDRESS + PATH_SEPARATOR + IndexCreationFactory.RAW); case MailboxTo : - return SortBuilders.fieldSort(JsonMessageConstants.TO + PATH_SEPARATOR + JsonMessageConstants.EMailer.ADDRESS - + PATH_SEPARATOR + IndexCreationFactory.RAW); + return new FieldSort.Builder().field(JsonMessageConstants.TO + PATH_SEPARATOR + + JsonMessageConstants.EMailer.ADDRESS + PATH_SEPARATOR + IndexCreationFactory.RAW); case BaseSubject : - return SortBuilders.fieldSort(JsonMessageConstants.SUBJECT + PATH_SEPARATOR + IndexCreationFactory.RAW); + return new FieldSort.Builder().field(JsonMessageConstants.SUBJECT + PATH_SEPARATOR + IndexCreationFactory.RAW); case Size : - return SortBuilders.fieldSort(JsonMessageConstants.SIZE); + return new FieldSort.Builder().field(JsonMessageConstants.SIZE); case SentDate : - return SortBuilders.fieldSort(JsonMessageConstants.SENT_DATE); + return new FieldSort.Builder().field(JsonMessageConstants.SENT_DATE); case Uid : - return SortBuilders.fieldSort(JsonMessageConstants.UID); + return new FieldSort.Builder().field(JsonMessageConstants.UID); case Id: - return SortBuilders.fieldSort(JsonMessageConstants.MESSAGE_ID); + return new FieldSort.Builder().field(JsonMessageConstants.MESSAGE_ID); default: throw new RuntimeException("Sort is not implemented"); } @@ -67,9 +67,9 @@ public class SortConverter { private static SortOrder getOrder(SearchQuery.Sort sort) { if (sort.isReverse()) { - return SortOrder.DESC; + return SortOrder.Desc; } else { - return SortOrder.ASC; + return SortOrder.Asc; } } } diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcher.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcher.java index f1bfcecfc6..7b7f4acd97 100644 --- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcher.java +++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcher.java @@ -22,6 +22,7 @@ package org.apache.james.mailbox.opensearch.search; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.james.backends.opensearch.AliasName; import org.apache.james.backends.opensearch.ReactorOpenSearchClient; @@ -32,16 +33,18 @@ import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.mailbox.opensearch.query.QueryConverter; import org.apache.james.mailbox.opensearch.query.SortConverter; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.search.Hit; + +import com.fasterxml.jackson.databind.node.ObjectNode; import reactor.core.publisher.Flux; public class OpenSearchSearcher { public static final int DEFAULT_SEARCH_SIZE = 100; - private static final TimeValue TIMEOUT = TimeValue.timeValueMinutes(1); + private static final Time TIMEOUT = new Time.Builder().time("1m").build(); private static final int MAX_ROUTING_KEY = 5; private final ReactorOpenSearchClient client; @@ -59,39 +62,40 @@ public class OpenSearchSearcher { this.routingKeyFactory = routingKeyFactory; } - public Flux<SearchHit> search(Collection<MailboxId> mailboxIds, SearchQuery query, - Optional<Integer> limit, List<String> fields) { + public Flux<Hit<ObjectNode>> search(Collection<MailboxId> mailboxIds, SearchQuery query, + Optional<Integer> limit, List<String> fields) { SearchRequest searchRequest = prepareSearch(mailboxIds, query, limit, fields); return new ScrolledSearch(client, searchRequest) .searchHits(); } private SearchRequest prepareSearch(Collection<MailboxId> mailboxIds, SearchQuery query, Optional<Integer> limit, List<String> fields) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .query(queryConverter.from(mailboxIds, query)) - .size(computeRequiredSize(limit)) - .storedFields(fields); - - query.getSorts() + List<SortOptions> sorts = query.getSorts() .stream() .map(SortConverter::convertSort) - .forEach(searchSourceBuilder::sort); + .map(fieldSort -> new SortOptions.Builder().field(fieldSort).build()) + .collect(Collectors.toList()); - SearchRequest request = new SearchRequest(aliasName.getValue()) + SearchRequest.Builder request = new SearchRequest.Builder() + .index(aliasName.getValue()) .scroll(TIMEOUT) - .source(searchSourceBuilder); + .query(queryConverter.from(mailboxIds, query)) + .size(computeRequiredSize(limit)) + .storedFields(fields) + .sort(sorts); return toRoutingKey(mailboxIds) .map(request::routing) - .orElse(request); + .orElse(request) + .build(); } - private Optional<String[]> toRoutingKey(Collection<MailboxId> mailboxIds) { + private Optional<String> toRoutingKey(Collection<MailboxId> mailboxIds) { if (mailboxIds.size() < MAX_ROUTING_KEY) { return Optional.of(mailboxIds.stream() .map(routingKeyFactory::from) .map(RoutingKey::asString) - .toArray(String[]::new)); + .collect(Collectors.joining(","))); } return Optional.empty(); } diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchIntegrationTest.java b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchIntegrationTest.java index 19f05f1c78..c87203b756 100644 --- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchIntegrationTest.java +++ b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchIntegrationTest.java @@ -63,11 +63,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch.core.SearchRequest; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -172,7 +170,7 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .setBody(Strings.repeat("0à2345678é", 3200), StandardCharsets.UTF_8)), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 14); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 14); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.address(SearchQuery.AddressType.To, recipient)), session)).toStream()) .containsExactly(composedMessageId.getUid()); @@ -193,11 +191,12 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { CALMLY_AWAIT.atMost(Durations.TEN_SECONDS) .untilAsserted(() -> assertThat(client.search( - new SearchRequest(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), - RequestOptions.DEFAULT) + new SearchRequest.Builder() + .index(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) + .query(QueryBuilders.matchAll().build()._toQuery()) + .build()) .block() - .getHits().getTotalHits().value).isGreaterThanOrEqualTo(14)); + .hits().total().value()).isEqualTo(14)); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.address(SearchQuery.AddressType.To, recipient)), session)).toStream()) .containsExactly(composedMessageId.getUid()); @@ -216,7 +215,7 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .setBody(Strings.repeat("0123456789 ", 5000), StandardCharsets.UTF_8)), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 14); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 14); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.bodyContains("0123456789")), session)).toStream()) .containsExactly(composedMessageId.getUid()); @@ -235,7 +234,7 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .setBody(Strings.repeat("0123456789 ", 5000) + " matchMe", StandardCharsets.UTF_8)), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 14); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 14); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.bodyContains("matchMe")), session)).toStream()) .containsExactly(composedMessageId.getUid()); @@ -277,7 +276,7 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .build(ClassLoaderUtils.getSystemResourceAsSharedStream("eml/mailCustomStringHeader.eml")), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 15); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 15); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.headerExists("Custom-header")), session)).toStream()) .containsExactly(customDateHeaderMessageId.getUid(), customStringHeaderMessageId.getUid()); @@ -330,7 +329,7 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .build()), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 15); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 15); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.address(SearchQuery.AddressType.To, "b...@other.tld")), session)).toStream()) .containsOnly(messageId2.getUid()); @@ -361,13 +360,19 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .build()), session).getId(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 15); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 15); Thread.sleep(500); assertThat(Flux.from(messageManager.search(SearchQuery.of(SearchQuery.address(SearchQuery.AddressType.To, "other")), session)).toStream()) .containsOnly(messageId2.getUid()); } + @Disabled("JAMES-3771 Waiting for a fix on opensearch client to be merged: https://github.com/opensearch-project/opensearch-java/pull/169") + @Test + public void sortOnCcShouldWork() throws Exception { + + } + @Disabled("MAILBOX-403 Relaxed the matching constraints for email addresses in text bodies to reduce OpenSearch disk space usage") @Test public void textShouldNotMatchOtherAddressesOfTheSameDomain() { @@ -470,13 +475,14 @@ class OpenSearchIntegrationTest extends AbstractMessageSearchIndexTest { .containsOnly(messageId1.getUid()); } - private void awaitForOpenSearch(QueryBuilder query, long totalHits) { + private void awaitForOpenSearch(Query query, long totalHits) { CALMLY_AWAIT.atMost(Durations.TEN_SECONDS) .untilAsserted(() -> assertThat(client.search( - new SearchRequest(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) - .source(new SearchSourceBuilder().query(query)), - RequestOptions.DEFAULT) + new SearchRequest.Builder() + .index(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) + .query(query) + .build()) .block() - .getHits().getTotalHits().value).isEqualTo(totalHits)); + .hits().total().value()).isEqualTo(totalHits)); } } \ No newline at end of file diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndexTest.java b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndexTest.java index ce34fbe26b..bf3025cbf2 100644 --- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndexTest.java +++ b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/events/OpenSearchListeningMessageSearchIndexTest.java @@ -86,11 +86,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch.core.SearchRequest; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -219,7 +218,7 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void addShouldIndexMessageWithoutAttachment() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -229,7 +228,7 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void addShouldIndexMessageWithAttachment() throws Exception { testee.add(session, mailbox, MESSAGE_WITH_ATTACHMENT).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -241,7 +240,7 @@ class OpenSearchListeningMessageSearchIndexTest { testee.add(session, mailbox, MESSAGE_1).block(); testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -253,7 +252,7 @@ class OpenSearchListeningMessageSearchIndexTest { testee.add(session, mailbox, MESSAGE_1).block(); testee.add(session, mailbox, MESSAGE_2).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 2L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 2L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -272,7 +271,7 @@ class OpenSearchListeningMessageSearchIndexTest { messageToOpenSearchJson, sessionProvider, new MailboxIdRoutingKeyFactory(), new InMemoryMessageId.Factory()); testee.add(session, mailbox, MESSAGE_WITH_ATTACHMENT).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -293,10 +292,10 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void deleteShouldRemoveIndex() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); testee.delete(session, mailbox.getMailboxId(), Lists.newArrayList(MESSAGE_UID_1)).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 0L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 0L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -307,10 +306,10 @@ class OpenSearchListeningMessageSearchIndexTest { void deleteShouldOnlyRemoveIndexesPassedAsArguments() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); testee.add(session, mailbox, MESSAGE_2).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 2L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 2L); testee.delete(session, mailbox.getMailboxId(), Lists.newArrayList(MESSAGE_UID_1)).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -321,10 +320,10 @@ class OpenSearchListeningMessageSearchIndexTest { void deleteShouldRemoveMultipleIndexes() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); testee.add(session, mailbox, MESSAGE_2).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 2L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 2L); testee.delete(session, mailbox.getMailboxId(), Lists.newArrayList(MESSAGE_UID_1, MESSAGE_UID_2)).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 0L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 0L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -334,11 +333,11 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void deleteShouldBeIdempotent() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); testee.delete(session, mailbox.getMailboxId(), Lists.newArrayList(MESSAGE_UID_1)).block(); testee.delete(session, mailbox.getMailboxId(), Lists.newArrayList(MESSAGE_UID_1)).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 0L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 0L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -365,7 +364,7 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void updateShouldUpdateIndex() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); Flags newFlags = new Flags(Flags.Flag.ANSWERED); UpdatedFlags updatedFlags = UpdatedFlags.builder() @@ -376,7 +375,7 @@ class OpenSearchListeningMessageSearchIndexTest { .build(); testee.update(session, mailbox.getMailboxId(), Lists.newArrayList(updatedFlags)).block(); - awaitForOpenSearch(QueryBuilders.termQuery("isAnswered", true), 1L); + awaitForOpenSearch(QueryBuilders.term().field("isAnswered").value(FieldValue.of(true)).build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.flagIsSet(Flags.Flag.ANSWERED)); assertThat(testee.search(session, mailbox, query).toStream()) @@ -384,9 +383,9 @@ class OpenSearchListeningMessageSearchIndexTest { } @Test - void updateShouldNotUpdateNorThrowOnUnknownMessageUid() throws Exception { + void updateShouldThrowOnUnknownMessageUid() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); Flags newFlags = new Flags(Flags.Flag.ANSWERED); UpdatedFlags updatedFlags = UpdatedFlags.builder() @@ -406,7 +405,7 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void updateShouldBeIdempotent() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 1L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 1L); Flags newFlags = new Flags(Flags.Flag.ANSWERED); UpdatedFlags updatedFlags = UpdatedFlags.builder() @@ -418,7 +417,7 @@ class OpenSearchListeningMessageSearchIndexTest { testee.update(session, mailbox.getMailboxId(), Lists.newArrayList(updatedFlags)).block(); testee.update(session, mailbox.getMailboxId(), Lists.newArrayList(updatedFlags)).block(); - awaitForOpenSearch(QueryBuilders.termQuery("isAnswered", true), 1L); + awaitForOpenSearch(QueryBuilders.term().field("isAnswered").value(FieldValue.of(true)).build()._toQuery(), 1L); SearchQuery query = SearchQuery.of(SearchQuery.flagIsSet(Flags.Flag.ANSWERED)); assertThat(testee.search(session, mailbox, query).toStream()) @@ -448,10 +447,10 @@ class OpenSearchListeningMessageSearchIndexTest { void deleteAllShouldRemoveAllIndexes() throws Exception { testee.add(session, mailbox, MESSAGE_1).block(); testee.add(session, mailbox, MESSAGE_2).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 2L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 2L); testee.deleteAll(session, mailbox.getMailboxId()).block(); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), 0L); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), 0L); SearchQuery query = SearchQuery.of(SearchQuery.all()); assertThat(testee.search(session, mailbox, query).toStream()) @@ -483,18 +482,21 @@ class OpenSearchListeningMessageSearchIndexTest { @Test void retrieveIndexedFlagsShouldReturnEmptyWhenNotFound() { + testee.add(session, mailbox, MESSAGE_1).block(); + assertThat(testee.retrieveIndexedFlags(mailbox, MESSAGE_UID_4).blockOptional()) .isEmpty(); } } - private void awaitForOpenSearch(QueryBuilder query, long totalHits) { + private void awaitForOpenSearch(Query query, long totalHits) { CALMLY_AWAIT.atMost(Durations.TEN_SECONDS) .untilAsserted(() -> assertThat(client.search( - new SearchRequest(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) - .source(new SearchSourceBuilder().query(query)), - RequestOptions.DEFAULT) + new SearchRequest.Builder() + .index(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) + .query(query) + .build()) .block() - .getHits().getTotalHits().value).isEqualTo(totalHits)); + .hits().total().value()).isEqualTo(totalHits)); } } \ No newline at end of file diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcherTest.java b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcherTest.java index 9c24556392..70fbde8e8b 100644 --- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcherTest.java +++ b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearcherTest.java @@ -67,11 +67,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch.core.SearchRequest; import com.github.fge.lambdas.Throwing; import com.google.common.collect.ImmutableList; @@ -157,7 +155,7 @@ class OpenSearchSearcherTest { .map(Throwing.<MailboxPath, ComposedMessageId>function(mailboxPath -> addMessage(session, mailboxPath)).sneakyThrow()) .collect(ImmutableList.toImmutableList()); - awaitForOpenSearch(QueryBuilders.matchAllQuery(), composedMessageIds.size()); + awaitForOpenSearch(QueryBuilders.matchAll().build()._toQuery(), composedMessageIds.size()); MultimailboxesSearchQuery multimailboxesSearchQuery = MultimailboxesSearchQuery .from(SearchQuery.of(SearchQuery.all())) @@ -184,13 +182,14 @@ class OpenSearchSearcherTest { .getId(); } - private void awaitForOpenSearch(QueryBuilder query, long totalHits) { + private void awaitForOpenSearch(Query query, long totalHits) { CALMLY_AWAIT.atMost(Durations.TEN_SECONDS) .untilAsserted(() -> assertThat(client.search( - new SearchRequest(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) - .source(new SearchSourceBuilder().query(query)), - RequestOptions.DEFAULT) + new SearchRequest.Builder() + .index(MailboxOpenSearchConstants.DEFAULT_MAILBOX_INDEX.getValue()) + .query(query) + .build()) .block() - .getHits().getTotalHits().value).isEqualTo(totalHits)); + .hits().total().value()).isEqualTo(totalHits)); } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org For additional commands, e-mail: notifications-h...@james.apache.org