This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 103d13846a JAMES-4057 Enable Query String to be used on OpenSearch
(#2704)
103d13846a is described below
commit 103d13846aaf7067fe71698db642678b747f9ee0
Author: Benoit TELLIER <[email protected]>
AuthorDate: Mon May 5 20:48:52 2025 +0200
JAMES-4057 Enable Query String to be used on OpenSearch (#2704)
This allows eg the user that both the terms "avocado" and "banana" are
expected when searching the string
"avocado banana"
Amongst overs.
---
.../opensearch/OpenSearchMailboxConfiguration.java | 23 +++-
.../opensearch/query/CriterionConverter.java | 149 +++++++++++++--------
.../OpenSearchNoIndexBodyIntegrationTest.java | 2 +-
...ionTest.java => OpenSearchQueryStringTest.java} | 39 +++---
4 files changed, 130 insertions(+), 83 deletions(-)
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
index c90d8ad806..6113ebcacd 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/OpenSearchMailboxConfiguration.java
@@ -38,6 +38,7 @@ public class OpenSearchMailboxConfiguration {
private Optional<IndexHeaders> indexHeaders;
private Optional<Boolean> optimiseMoves;
private Optional<Boolean> textFuzzinessSearch;
+ private Optional<Boolean> useQueryStringQuery;
private Optional<IndexBody> indexBody;
private Optional<IndexUser> indexUser;
@@ -49,6 +50,7 @@ public class OpenSearchMailboxConfiguration {
indexHeaders = Optional.empty();
optimiseMoves = Optional.empty();
textFuzzinessSearch = Optional.empty();
+ useQueryStringQuery = Optional.empty();
indexBody = Optional.empty();
indexUser = Optional.empty();
}
@@ -88,6 +90,12 @@ public class OpenSearchMailboxConfiguration {
return this;
}
+
+ public Builder useQueryStringQuery(Boolean useQueryStringQuery) {
+ this.useQueryStringQuery =
Optional.ofNullable(useQueryStringQuery);
+ return this;
+ }
+
public Builder indexBody(IndexBody indexBody) {
this.indexBody = Optional.ofNullable(indexBody);
return this;
@@ -107,6 +115,7 @@ public class OpenSearchMailboxConfiguration {
indexHeaders.orElse(IndexHeaders.YES),
optimiseMoves.orElse(DEFAULT_OPTIMIZE_MOVES),
textFuzzinessSearch.orElse(DEFAULT_TEXT_FUZZINESS_SEARCH),
+ useQueryStringQuery.orElse(DEFAULT_USE_SIMPLE_TEXT_QUERY),
indexBody.orElse(IndexBody.YES),
indexUser.orElse(IndexUser.NO));
}
@@ -126,12 +135,14 @@ public class OpenSearchMailboxConfiguration {
private static final String OPENSEARCH_INDEX_HEADERS =
"opensearch.indexHeaders";
private static final String OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE =
"opensearch.message.index.optimize.move";
private static final String OPENSEARCH_TEXT_FUZZINESS_SEARCH =
"opensearch.text.fuzziness.search";
+ private static final String OPENSEARCH_TEXT_STRING_QUERY =
"opensearch.text.string.query";
private static final String OPENSEARCH_INDEX_BODY = "opensearch.indexBody";
private static final String OPENSEARCH_INDEX_USER = "opensearch.indexUser";
private static final boolean DEFAULT_INDEX_ATTACHMENTS = true;
private static final boolean DEFAULT_INDEX_HEADERS = true;
public static final boolean DEFAULT_OPTIMIZE_MOVES = false;
public static final boolean DEFAULT_TEXT_FUZZINESS_SEARCH = false;
+ public static final boolean DEFAULT_USE_SIMPLE_TEXT_QUERY = false;
public static final boolean DEFAULT_INDEX_BODY = true;
public static final boolean DEFAULT_INDEX_USER = false;
public static final OpenSearchMailboxConfiguration DEFAULT_CONFIGURATION =
builder().build();
@@ -145,6 +156,7 @@ public class OpenSearchMailboxConfiguration {
.indexHeaders(provideIndexHeaders(configuration))
.optimiseMoves(configuration.getBoolean(OPENSEARCH_MESSAGE_INDEX_OPTIMIZE_MOVE,
null))
.textFuzzinessSearch(configuration.getBoolean(OPENSEARCH_TEXT_FUZZINESS_SEARCH,
null))
+
.useQueryStringQuery(configuration.getBoolean(OPENSEARCH_TEXT_STRING_QUERY,
null))
.indexBody(provideIndexBody(configuration))
.indexUser(provideIndexUser(configuration))
.build();
@@ -206,12 +218,13 @@ public class OpenSearchMailboxConfiguration {
private final IndexHeaders indexHeaders;
private final boolean optimiseMoves;
private final boolean textFuzzinessSearch;
+ private final boolean useQueryStringQuery;
private final IndexBody indexBody;
private final IndexUser indexUser;
private OpenSearchMailboxConfiguration(IndexName indexMailboxName,
ReadAliasName readAliasMailboxName,
WriteAliasName
writeAliasMailboxName, IndexAttachments indexAttachment,
- IndexHeaders indexHeaders, boolean
optimiseMoves, boolean textFuzzinessSearch,
+ IndexHeaders indexHeaders, boolean
optimiseMoves, boolean textFuzzinessSearch, boolean useSimpleTextQuery,
IndexBody indexBody, IndexUser
indexUser) {
this.indexMailboxName = indexMailboxName;
this.readAliasMailboxName = readAliasMailboxName;
@@ -220,6 +233,7 @@ public class OpenSearchMailboxConfiguration {
this.indexHeaders = indexHeaders;
this.optimiseMoves = optimiseMoves;
this.textFuzzinessSearch = textFuzzinessSearch;
+ this.useQueryStringQuery = useSimpleTextQuery;
this.indexBody = indexBody;
this.indexUser = indexUser;
}
@@ -260,6 +274,10 @@ public class OpenSearchMailboxConfiguration {
return indexUser;
}
+ public boolean isUseQueryStringQuery() {
+ return useQueryStringQuery;
+ }
+
@Override
public final boolean equals(Object o) {
if (o instanceof OpenSearchMailboxConfiguration) {
@@ -271,6 +289,7 @@ public class OpenSearchMailboxConfiguration {
&& Objects.equals(this.readAliasMailboxName,
that.readAliasMailboxName)
&& Objects.equals(this.optimiseMoves, that.optimiseMoves)
&& Objects.equals(this.textFuzzinessSearch,
that.textFuzzinessSearch)
+ && Objects.equals(this.useQueryStringQuery,
that.useQueryStringQuery)
&& Objects.equals(this.writeAliasMailboxName,
that.writeAliasMailboxName)
&& Objects.equals(this.indexBody, that.indexBody)
&& Objects.equals(this.indexUser, that.indexUser);
@@ -281,6 +300,6 @@ public class OpenSearchMailboxConfiguration {
@Override
public final int hashCode() {
return Objects.hash(indexMailboxName, readAliasMailboxName,
writeAliasMailboxName, indexAttachment, indexHeaders,
- writeAliasMailboxName, optimiseMoves, textFuzzinessSearch,
indexBody, indexUser);
+ writeAliasMailboxName, optimiseMoves, textFuzzinessSearch,
useQueryStringQuery, indexBody, indexUser);
}
}
diff --git
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
index 714d53bee3..a85a1c050b 100644
---
a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
+++
b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
@@ -49,22 +49,27 @@ import
org.opensearch.client.opensearch._types.query_dsl.MatchQuery;
import org.opensearch.client.opensearch._types.query_dsl.NestedQuery;
import org.opensearch.client.opensearch._types.query_dsl.Operator;
import org.opensearch.client.opensearch._types.query_dsl.Query;
+import org.opensearch.client.opensearch._types.query_dsl.QueryStringQuery;
import org.opensearch.client.opensearch._types.query_dsl.RangeQuery;
+import
org.opensearch.client.opensearch._types.query_dsl.SimpleQueryStringQuery;
import org.opensearch.client.opensearch._types.query_dsl.TermQuery;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
public class CriterionConverter {
private final Map<Class<?>, Function<Criterion, Query>>
criterionConverterMap;
private final Map<Class<?>, BiFunction<String, HeaderOperator, Query>>
headerOperatorConverterMap;
private final String textFuzzinessSearchValue;
+ private final boolean useQueryStringQuery;
@Inject
public CriterionConverter(OpenSearchMailboxConfiguration
openSearchMailboxConfiguration) {
this.criterionConverterMap = new HashMap<>();
this.headerOperatorConverterMap = new HashMap<>();
this.textFuzzinessSearchValue =
evaluateFuzzinessValue(openSearchMailboxConfiguration.textFuzzinessSearchEnable());
+ this.useQueryStringQuery =
openSearchMailboxConfiguration.isUseQueryStringQuery();
registerCriterionConverters();
registerHeaderOperatorConverters();
@@ -75,6 +80,7 @@ public class CriterionConverter {
this.criterionConverterMap = new HashMap<>();
this.headerOperatorConverterMap = new HashMap<>();
this.textFuzzinessSearchValue =
evaluateFuzzinessValue(DEFAULT_TEXT_FUZZINESS_SEARCH);
+ this.useQueryStringQuery = false;
registerCriterionConverters();
registerHeaderOperatorConverters();
@@ -218,59 +224,80 @@ public class CriterionConverter {
private Query convertTextCriterion(SearchQuery.TextCriterion
textCriterion) {
switch (textCriterion.getType()) {
case BODY:
- return new BoolQuery.Builder()
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.TEXT_BODY)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
- .build()
- .toQuery())
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.HTML_BODY)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
+ if (useQueryStringQuery) {
+ return new SimpleQueryStringQuery.Builder()
+ .fields(ImmutableList.of(JsonMessageConstants.TEXT_BODY,
JsonMessageConstants.HTML_BODY))
+ .query(textCriterion.getOperator().getValue())
+ .build().toQuery();
+ } else {
+ return new BoolQuery.Builder()
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.TEXT_BODY)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.HTML_BODY)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
.build()
- .toQuery())
- .build()
- .toQuery();
+ .toQuery();
+ }
case FULL:
- return new BoolQuery.Builder()
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.TEXT_BODY)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
- .build()
- .toQuery())
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.HTML_BODY)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
- .build()
- .toQuery())
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.ATTACHMENTS + "." +
JsonMessageConstants.Attachment.TEXT_CONTENT)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
+ if (useQueryStringQuery) {
+ return new SimpleQueryStringQuery.Builder()
+ .fields(ImmutableList.of(JsonMessageConstants.TEXT_BODY,
JsonMessageConstants.HTML_BODY, JsonMessageConstants.ATTACHMENTS + "." +
JsonMessageConstants.Attachment.TEXT_CONTENT))
+ .query(textCriterion.getOperator().getValue())
+ .build().toQuery();
+ } else {
+ return new BoolQuery.Builder()
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.TEXT_BODY)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.HTML_BODY)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.ATTACHMENTS + "." +
JsonMessageConstants.Attachment.TEXT_CONTENT)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
.build()
- .toQuery())
- .build()
- .toQuery();
+ .toQuery();
+ }
case ATTACHMENTS:
- return new BoolQuery.Builder()
- .should(new MatchQuery.Builder()
- .field(JsonMessageConstants.ATTACHMENTS + "." +
JsonMessageConstants.Attachment.TEXT_CONTENT)
- .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
+ if (useQueryStringQuery) {
+ return new SimpleQueryStringQuery.Builder()
+ .fields(ImmutableList.of(JsonMessageConstants.ATTACHMENTS
+ "." + JsonMessageConstants.Attachment.TEXT_CONTENT))
+ .query(textCriterion.getOperator().getValue())
+ .build().toQuery();
+ } else {
+ return new BoolQuery.Builder()
+ .should(new MatchQuery.Builder()
+ .field(JsonMessageConstants.ATTACHMENTS + "." +
JsonMessageConstants.Attachment.TEXT_CONTENT)
+ .query(new
FieldValue.Builder().stringValue(textCriterion.getOperator().getValue()).build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery())
.build()
- .toQuery())
- .build()
- .toQuery();
+ .toQuery();
+ }
case ATTACHMENT_FILE_NAME:
return new BoolQuery.Builder()
.should(new MatchQuery.Builder()
@@ -466,15 +493,23 @@ public class CriterionConverter {
}
private Query convertSubject(SearchQuery.SubjectCriterion headerCriterion)
{
- return new MatchQuery.Builder()
- .field(JsonMessageConstants.SUBJECT)
- .query(new FieldValue.Builder()
- .stringValue(headerCriterion.getSubject())
- .build())
- .fuzziness(textFuzzinessSearchValue)
- .operator(Operator.And)
- .build()
- .toQuery();
+ if (useQueryStringQuery) {
+ return new QueryStringQuery.Builder()
+ .fields(ImmutableList.of(JsonMessageConstants.SUBJECT))
+ .query(headerCriterion.getSubject())
+ .fuzziness(textFuzzinessSearchValue)
+ .build().toQuery();
+ } else {
+ return new MatchQuery.Builder()
+ .field(JsonMessageConstants.SUBJECT)
+ .query(new FieldValue.Builder()
+ .stringValue(headerCriterion.getSubject())
+ .build())
+ .fuzziness(textFuzzinessSearchValue)
+ .operator(Operator.And)
+ .build()
+ .toQuery();
+ }
}
private Query manageAddressFields(String headerName, String value) {
diff --git
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
index 8673fe901f..72b42ec897 100644
---
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
+++
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
@@ -71,7 +71,7 @@ import com.google.common.collect.ImmutableSet;
import reactor.core.publisher.Mono;
-public class OpenSearchNoIndexBodyIntegrationTest {
+class OpenSearchNoIndexBodyIntegrationTest {
static final int SEARCH_SIZE = 1;
private static final Username USERNAME = Username.of("user");
private static final ConditionFactory CALMLY_AWAIT = Awaitility
diff --git
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchQueryStringTest.java
similarity index 89%
copy from
mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
copy to
mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchQueryStringTest.java
index 8673fe901f..74b3f41a25 100644
---
a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchNoIndexBodyIntegrationTest.java
+++
b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/OpenSearchQueryStringTest.java
@@ -71,8 +71,8 @@ import com.google.common.collect.ImmutableSet;
import reactor.core.publisher.Mono;
-public class OpenSearchNoIndexBodyIntegrationTest {
- static final int SEARCH_SIZE = 1;
+class OpenSearchQueryStringTest {
+ static final int SEARCH_SIZE = 10;
private static final Username USERNAME = Username.of("user");
private static final ConditionFactory CALMLY_AWAIT = Awaitility
.with().pollInterval(ONE_HUNDRED_MILLISECONDS)
@@ -113,6 +113,10 @@ public class OpenSearchNoIndexBodyIntegrationTest {
MailboxIndexCreationUtil.prepareClient(client, readAliasName,
writeAliasName, indexName,
openSearch.getDockerOpenSearch().configuration());
+ OpenSearchMailboxConfiguration openSearchMailboxConfiguration =
OpenSearchMailboxConfiguration.builder()
+ .indexBody(IndexBody.YES)
+ .useQueryStringQuery(true)
+ .build();
InMemoryIntegrationResources resources =
InMemoryIntegrationResources.builder()
.preProvisionnedFakeAuthenticator()
.fakeAuthorizator()
@@ -123,10 +127,10 @@ public class OpenSearchNoIndexBodyIntegrationTest {
preInstanciationStage.getMapperFactory(),
ImmutableSet.of(),
new OpenSearchIndexer(client, writeAliasName),
- new OpenSearchSearcher(client, new QueryConverter(new
CriterionConverter()), SEARCH_SIZE, readAliasName, routingKeyFactory),
- new MessageToOpenSearchJson(textExtractor,
ZoneId.of("Europe/Paris"), IndexAttachments.YES, IndexHeaders.YES,
IndexBody.NO),
+ new OpenSearchSearcher(client, new QueryConverter(new
CriterionConverter(openSearchMailboxConfiguration)), SEARCH_SIZE,
readAliasName, routingKeyFactory),
+ new MessageToOpenSearchJson(textExtractor,
ZoneId.of("Europe/Paris"), IndexAttachments.YES, IndexHeaders.YES,
IndexBody.YES),
preInstanciationStage.getSessionProvider(), routingKeyFactory,
messageIdFactory,
-
OpenSearchMailboxConfiguration.builder().indexBody(IndexBody.NO).build(), new
RecordingMetricFactory(),
+ openSearchMailboxConfiguration, new RecordingMetricFactory(),
ImmutableSet.of()))
.noPreDeletionHooks()
.storeQuotaManager()
@@ -150,37 +154,26 @@ public class OpenSearchNoIndexBodyIntegrationTest {
@Test
void searchingByBodyContentShouldNotReturnMessageWhenNoIndexBody() throws
Exception {
- addMessage(session, inboxPath).block();
-
- awaitForOpenSearch(QueryBuilders.matchAll().build().toQuery(), 1L);
-
- SearchQuery searchQuery =
SearchQuery.of(SearchQuery.bodyContains("Hello"));
-
- assertThat(messageSearchIndex.search(session, mailbox,
searchQuery).collectList().block())
- .isEmpty();
- }
-
- @Test
- void searchingAllShoulReturnMessageWhenNoIndexBody() throws Exception {
- ComposedMessageId composedMessageId = addMessage(session,
inboxPath).block();
+ ComposedMessageId expectedId = addMessage(session, inboxPath, "Lucas
love avocado banana smoothie").block();
+ addMessage(session, inboxPath, "Avocado grows in Mexico").block();
- awaitForOpenSearch(QueryBuilders.matchAll().build().toQuery(), 1L);
+ awaitForOpenSearch(QueryBuilders.matchAll().build().toQuery(), 2L);
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.all());
+ SearchQuery searchQuery =
SearchQuery.of(SearchQuery.bodyContains("\"love avocado\""));
assertThat(messageSearchIndex.search(session, mailbox,
searchQuery).collectList().block())
- .containsExactly(composedMessageId.getUid());
+ .containsOnly(expectedId.getUid());
}
- private Mono<ComposedMessageId> addMessage(MailboxSession session,
MailboxPath mailboxPath) throws Exception {
+ private Mono<ComposedMessageId> addMessage(MailboxSession session,
MailboxPath mailboxPath, String message) throws Exception {
MessageManager messageManager =
storeMailboxManager.getMailbox(mailboxPath, session);
String recipient = "[email protected]";
return
Mono.from(messageManager.appendMessageReactive(MessageManager.AppendCommand.from(
Message.Builder.of()
.setTo(recipient)
- .setBody("Hello", StandardCharsets.UTF_8)),
+ .setBody(message, StandardCharsets.UTF_8)),
session))
.map(MessageManager.AppendResult::getId);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
