JAMES-2555 Proves our fix helps reindexing big mailboxes Before the fix the added test was failing with OutOfMemory exception after 300 messages. Now it passes easily 1000 messages.
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/962683d3 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/962683d3 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/962683d3 Branch: refs/heads/master Commit: 962683d38715e7a40286e422fc0487c495a47c32 Parents: 12a3445 Author: Benoit Tellier <[email protected]> Authored: Thu Oct 11 09:19:44 2018 +0700 Committer: Benoit Tellier <[email protected]> Committed: Fri Oct 12 15:27:34 2018 +0700 ---------------------------------------------------------------------- mailbox/tools/indexer/pom.xml | 53 ++++++++- .../indexer/CassandraReIndexerImplTest.java | 119 +++++++++++++++++++ 2 files changed, 170 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/962683d3/mailbox/tools/indexer/pom.xml ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/pom.xml b/mailbox/tools/indexer/pom.xml index e0874f0..4395bba 100644 --- a/mailbox/tools/indexer/pom.xml +++ b/mailbox/tools/indexer/pom.xml @@ -34,6 +34,17 @@ <dependencies> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>apache-james-backends-cassandra</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>apache-james-backends-cassandra</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>apache-james-mailbox-api</artifactId> </dependency> <dependency> @@ -44,11 +55,12 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> - <artifactId>apache-james-mailbox-store</artifactId> + <artifactId>apache-james-mailbox-cassandra</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>${james.groupId}</groupId> - <artifactId>apache-james-mailbox-store</artifactId> + <artifactId>apache-james-mailbox-cassandra</artifactId> <type>test-jar</type> <scope>test</scope> </dependency> @@ -64,6 +76,26 @@ <scope>test</scope> </dependency> <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>apache-james-mailbox-store</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>apache-james-mailbox-store</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>blob-cassandra</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <scope>test</scope> @@ -91,7 +123,24 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <scope>test</scope> + </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <argLine>-Xms128m -Xmx256m</argLine> + <reuseForks>false</reuseForks> + </configuration> + </plugin> + </plugins> + </build> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/962683d3/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java ---------------------------------------------------------------------- diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java new file mode 100644 index 0000000..7af0646 --- /dev/null +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java @@ -0,0 +1,119 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.mailbox.tools.indexer; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; + +import org.apache.james.backends.cassandra.CassandraCluster; +import org.apache.james.backends.cassandra.DockerCassandraRule; +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.MessageManager; +import org.apache.james.mailbox.cassandra.CassandraMailboxManager; +import org.apache.james.mailbox.cassandra.CassandraMailboxManagerProvider; +import org.apache.james.mailbox.cassandra.mail.MailboxAggregateModule; +import org.apache.james.mailbox.indexer.ReIndexer; +import org.apache.james.mailbox.model.MailboxPath; +import org.apache.james.mailbox.store.MailboxSessionMapperFactory; +import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; +import org.apache.james.util.concurrency.ConcurrentTestRunner; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import com.google.common.base.Strings; + +public class CassandraReIndexerImplTest { + private static final String USERNAME = "[email protected]"; + public static final MailboxPath INBOX = MailboxPath.forUser(USERNAME, "INBOX"); + private CassandraMailboxManager mailboxManager; + private ListeningMessageSearchIndex messageSearchIndex; + + private ReIndexer reIndexer; + + @ClassRule + public static DockerCassandraRule cassandraServer = new DockerCassandraRule(); + + private static CassandraCluster cassandra; + + @BeforeClass + public static void setUpClass() { + cassandra = CassandraCluster.create(MailboxAggregateModule.MODULE_WITH_QUOTA, cassandraServer.getHost()); + } + + @Before + public void setUp() { + mailboxManager = CassandraMailboxManagerProvider.provideMailboxManager(cassandra.getConf(), cassandra.getTypesProvider()); + MailboxSessionMapperFactory mailboxSessionMapperFactory = mailboxManager.getMapperFactory(); + messageSearchIndex = mock(ListeningMessageSearchIndex.class); + reIndexer = new ReIndexerImpl(mailboxManager, messageSearchIndex, mailboxSessionMapperFactory); + } + + @After + public void tearDown() { + cassandra.clearTables(); + } + + @AfterClass + public static void tearDownClass() { + cassandra.closeCluster(); + } + + @Test + public void reIndexShouldBeWellPerformed() throws Exception { + // Given a mailbox with 1000 messages * 150 KB + MailboxSession systemSession = mailboxManager.createSystemSession(USERNAME); + mailboxManager.createMailbox(INBOX, systemSession); + + byte[] bigBody = (Strings.repeat("header: value\r\n", 10000) + "\r\nbody").getBytes(StandardCharsets.UTF_8); + + int threadCount = 10; + int operationCount = 100; + MessageManager mailbox = mailboxManager.getMailbox(INBOX, systemSession); + ConcurrentTestRunner.builder() + .operation((a, b) -> mailbox + .appendMessage( + MessageManager.AppendCommand.builder().build(bigBody), + systemSession)) + .threadCount(threadCount) + .operationCount(operationCount) + .runSuccessfullyWithin(Duration.ofMinutes(10)); + + // When We re-index + reIndexer.reIndex(INBOX); + + // The indexer is called for each message + verify(messageSearchIndex).deleteAll(any(MailboxSession.class), any(Mailbox.class)); + verify(messageSearchIndex, times(threadCount * operationCount)) + .add(any(MailboxSession.class), any(Mailbox.class),any(MailboxMessage.class)); + verifyNoMoreInteractions(messageSearchIndex); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
