MAILBOX-266 Set the size of SearchResponses to 1 in ElasticSearch tests This allows to ensure Scrolls are well performed
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/b99d0c7f Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/b99d0c7f Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/b99d0c7f Branch: refs/heads/master Commit: b99d0c7f0164e431a4cbaa3e8e8fc19030ba3a6a Parents: 6ba31a6 Author: Benoit Tellier <btell...@linagora.com> Authored: Mon Mar 28 10:26:36 2016 +0700 Committer: Benoit Tellier <btell...@linagora.com> Committed: Wed Apr 6 16:18:45 2016 +0700 ---------------------------------------------------------------------- .../mailbox/elasticsearch/DeleteByQueryPerformer.java | 13 +++++++++++-- .../elasticsearch/search/ElasticSearchSearcher.java | 12 ++++++++++-- .../elasticsearch/ElasticSearchIndexerTest.java | 3 ++- .../elasticsearch/ElasticSearchIntegrationTest.java | 6 ++++-- 4 files changed, 27 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java index 9d266ce..895f016 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java @@ -34,17 +34,26 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchHit; +import com.google.common.annotations.VisibleForTesting; + public class DeleteByQueryPerformer { - public static final int BATCH_SIZE = 100; + public static final int DEFAULT_BATCH_SIZE = 100; public static final TimeValue TIMEOUT = new TimeValue(60000); private final ClientProvider clientProvider; private final ExecutorService executor; + private final int batchSize; @Inject public DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor") ExecutorService executor) { + this(clientProvider, executor, DEFAULT_BATCH_SIZE); + } + + @VisibleForTesting + DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor") ExecutorService executor, int batchSize) { this.clientProvider = clientProvider; this.executor = executor; + this.batchSize = batchSize; } public void perform(QueryBuilder queryBuilder) { @@ -59,7 +68,7 @@ public class DeleteByQueryPerformer { .setScroll(TIMEOUT) .setNoFields() .setQuery(queryBuilder) - .setSize(BATCH_SIZE)); + .setSize(batchSize)); for (SearchResponse searchResponse : scrollIterable) { deleteRetrievedIds(client, searchResponse); } http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java index 96d90b5..ab4698e 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java @@ -46,14 +46,22 @@ import org.slf4j.LoggerFactory; public class ElasticSearchSearcher<Id extends MailboxId> { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchSearcher.class); + private static final TimeValue TIMEOUT = new TimeValue(60000); + public static final int DEFAULT_SIZE = 100; private final ClientProvider clientProvider; private final QueryConverter queryConverter; + private final int size; @Inject public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter) { + this(clientProvider, queryConverter, DEFAULT_SIZE); + } + + public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter, int size) { this.clientProvider = clientProvider; this.queryConverter = queryConverter; + this.size = size; } public Iterator<Long> search(Mailbox<Id> mailbox, SearchQuery searchQuery) throws MailboxException { @@ -70,9 +78,9 @@ public class ElasticSearchSearcher<Id extends MailboxId> { .reduce( client.prepareSearch(ElasticSearchIndexer.MAILBOX_INDEX) .setTypes(ElasticSearchIndexer.MESSAGE_TYPE) - .setScroll(new TimeValue(60000)) + .setScroll(TIMEOUT) .setQuery(queryConverter.from(searchQuery, mailbox.getMailboxId().serialize())) - .setSize(100), + .setSize(size), (searchBuilder, sort) -> searchBuilder.addSort(SortConverter.convertSort(sort)), (partialResult1, partialResult2) -> partialResult1); } http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java index 1e4ce1a..7501c57 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java @@ -41,6 +41,7 @@ import com.google.common.collect.Lists; public class ElasticSearchIndexerTest { + public static final int MINIMUM_BATCH_SIZE = 1; private TemporaryFolder temporaryFolder = new TemporaryFolder(); private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder); @@ -55,7 +56,7 @@ public class ElasticSearchIndexerTest { public void setup() throws IOException { node = embeddedElasticSearch.getNode(); TestingClientProvider clientProvider = new TestingClientProvider(node); - deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor()); + deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor(), MINIMUM_BATCH_SIZE); testee = new ElasticSearchIndexer(clientProvider, deleteByQueryPerformer); } http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java index 2f5962e..9b90b72 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java @@ -62,6 +62,8 @@ import com.google.common.collect.Lists; public class ElasticSearchIntegrationTest { private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchIntegrationTest.class); + public static final int BATCH_SIZE = 1; + public static final int SEARCH_SIZE = 1; private TemporaryFolder temporaryFolder = new TemporaryFolder(); private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder); @@ -168,8 +170,8 @@ public class ElasticSearchIntegrationTest { ); MailboxSessionMapperFactory<InMemoryId> mapperFactory = new InMemoryMailboxSessionMapperFactory(); elasticSearchListeningMessageSearchIndex = new ElasticSearchListeningMessageSearchIndex<>(mapperFactory, - new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor())), - new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter())), + new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor(), BATCH_SIZE)), + new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter()), SEARCH_SIZE), new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"))); storeMailboxManager = new StoreMailboxManager<>( mapperFactory, --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org