This is an automated email from the ASF dual-hosted git repository. rcordier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit af777451819bc12b1b1e789c33447b294e3b74ab Author: Benoit Tellier <[email protected]> AuthorDate: Fri Jul 31 10:51:32 2020 +0700 JAMES-3317 Add explicit integration tests for pass-through & deduplication blobStore choosing On top of Cassandra blobStore, and Distributed Server: - Verify explicitly that blobs works on top of pass-through & deduplication options - Verify that deduplication do store same blobs together - Verify that pass-trough stores each blob separately --- .../modules/blobstore/BlobStoreConfiguration.java | 7 +- .../james/CassandraRabbitMQJamesServerFixture.java | 36 +++--- .../james/RabbitMQJamesServerReprocessingTest.java | 36 +++--- ...RabbitMQJamesServerWithRetryConnectionTest.java | 36 +++--- .../james/WithCassandraBlobStoreImmutableTest.java | 48 -------- .../james/WithCassandraBlobStoreMutableTest.java | 29 ----- ...ssandraDeduplicationBlobStoreImmutableTest.java | 45 ++++++++ ...CassandraDeduplicationBlobStoreMutableTest.java | 127 +++++++++++++++++++++ ...CassandraPassThroughBlobStoreImmutableTest.java | 45 ++++++++ ...thCassandraPassThroughBlobStoreMutableTest.java | 127 +++++++++++++++++++++ .../james/WithDefaultAwsS3ImmutableTest.java | 36 +++--- .../james/WithDefaultSwiftImmutableTest.java | 36 +++--- .../james/WithEncryptedAwsS3ImmutableTest.java | 36 +++--- .../james/WithEncryptedSwiftImmutableTest.java | 36 +++--- .../apache/james/modules/DockerRabbitMQRule.java | 36 +++--- .../apache/james/modules/RabbitMQExtension.java | 36 +++--- .../apache/james/modules/TestRabbitMQModule.java | 36 +++--- 17 files changed, 528 insertions(+), 260 deletions(-) diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreConfiguration.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreConfiguration.java index 7507794..be9a220 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreConfiguration.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreConfiguration.java @@ -164,11 +164,12 @@ public class BlobStoreConfiguration { } @VisibleForTesting - public static BlobStoreConfiguration cassandra() { - return new BlobStoreConfiguration(BlobStoreImplName.CASSANDRA, !CACHE_ENABLED, StorageStrategy.PASSTHROUGH); + public static RequireStoringStrategy cassandra() { + return builder() + .cassandra() + .disableCache(); } - private final BlobStoreImplName implementation; private final boolean cacheEnabled; private final StorageStrategy storageStrategy; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java index a07800b..2027540 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerReprocessingTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerReprocessingTest.java index d383cc5..b5b60aa 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerReprocessingTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerReprocessingTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerWithRetryConnectionTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerWithRetryConnectionTest.java index da1a615..3a6b1c5 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerWithRetryConnectionTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/RabbitMQJamesServerWithRetryConnectionTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreImmutableTest.java deleted file mode 100644 index cb3a5f1..0000000 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreImmutableTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.james; - -import org.apache.james.jmap.draft.JmapJamesServerContract; -import org.apache.james.modules.RabbitMQExtension; -import org.apache.james.modules.TestJMAPServerModule; -import org.apache.james.modules.blobstore.BlobStoreConfiguration; -import org.junit.jupiter.api.extension.RegisterExtension; - -public class WithCassandraBlobStoreImmutableTest implements JmapJamesServerContract, JamesServerContract { - @RegisterExtension - static JamesServerExtension jamesServerExtension = baseExtensionBuilder() - .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) - .build(); - - static JamesServerBuilder<CassandraRabbitMQJamesConfiguration> baseExtensionBuilder() { - return new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir -> - CassandraRabbitMQJamesConfiguration.builder() - .workingDirectory(tmpDir) - .configurationFromClasspath() - .blobStore(BlobStoreConfiguration.cassandra()) - .searchConfiguration(SearchConfiguration.elasticSearch()) - .build()) - .server(configuration -> CassandraRabbitMQJamesServerMain.createServer(configuration) - .overrideWith(new TestJMAPServerModule())) - .extension(new DockerElasticSearchExtension()) - .extension(new CassandraExtension()) - .extension(new RabbitMQExtension()); - } -} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreMutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreMutableTest.java deleted file mode 100644 index cbc908f..0000000 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreMutableTest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.james; - -import org.junit.jupiter.api.extension.RegisterExtension; - -public class WithCassandraBlobStoreMutableTest implements MailsShouldBeWellReceived { - @RegisterExtension - static JamesServerExtension jamesServerExtension = WithCassandraBlobStoreImmutableTest.baseExtensionBuilder() - .lifeCycle(JamesServerExtension.Lifecycle.PER_TEST) - .build(); -} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreImmutableTest.java new file mode 100644 index 0000000..f18bfcf --- /dev/null +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreImmutableTest.java @@ -0,0 +1,45 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james; + +import org.apache.james.jmap.draft.JmapJamesServerContract; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.TestJMAPServerModule; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.junit.jupiter.api.extension.RegisterExtension; + +public class WithCassandraDeduplicationBlobStoreImmutableTest implements JmapJamesServerContract, JamesServerContract { + @RegisterExtension + static JamesServerExtension jamesServerExtension = new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir -> + CassandraRabbitMQJamesConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.cassandra() + .deduplication()) + .searchConfiguration(SearchConfiguration.elasticSearch()) + .build()) + .server(configuration -> CassandraRabbitMQJamesServerMain.createServer(configuration) + .overrideWith(new TestJMAPServerModule())) + .extension(new DockerElasticSearchExtension()) + .extension(new CassandraExtension()) + .extension(new RabbitMQExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); +} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreMutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreMutableTest.java new file mode 100644 index 0000000..655795e --- /dev/null +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraDeduplicationBlobStoreMutableTest.java @@ -0,0 +1,127 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james; + +import static com.datastax.driver.core.querybuilder.QueryBuilder.select; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; + +import org.apache.james.backends.cassandra.init.ClusterFactory; +import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration; +import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration; +import org.apache.james.blob.cassandra.BlobTables; +import org.apache.james.core.Domain; +import org.apache.james.mailbox.DefaultMailboxes; +import org.apache.james.modules.MailboxProbeImpl; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.TestJMAPServerModule; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.modules.protocols.ImapGuiceProbe; +import org.apache.james.modules.protocols.SmtpGuiceProbe; +import org.apache.james.server.CassandraProbe; +import org.apache.james.util.Port; +import org.apache.james.utils.DataProbeImpl; +import org.apache.james.utils.SMTPMessageSender; +import org.apache.james.utils.SpoolerProbe; +import org.apache.james.utils.TestIMAPClient; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; + +public class WithCassandraDeduplicationBlobStoreMutableTest implements MailsShouldBeWellReceived { + private static final String JAMES_SERVER_HOST = "127.0.0.1"; + private static final String YET_ANOTHER_USER = "yet-another-user@" + DOMAIN; + + @RegisterExtension + static JamesServerExtension jamesServerExtension = new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir -> + CassandraRabbitMQJamesConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.cassandra() + .deduplication()) + .searchConfiguration(SearchConfiguration.elasticSearch()) + .build()) + .server(configuration -> CassandraRabbitMQJamesServerMain.createServer(configuration) + .overrideWith(new TestJMAPServerModule())) + .extension(new DockerElasticSearchExtension()) + .extension(new CassandraExtension()) + .extension(new RabbitMQExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_TEST) + .build(); + + @Test + void blobsShouldBeDeduplicated(GuiceJamesServer server) throws Exception { + server.getProbe(DataProbeImpl.class).fluent() + .addDomain(DOMAIN) + .addUser(JAMES_USER, PASSWORD) + .addUser(OTHER_USER, PASSWORD_OTHER) + .addUser(YET_ANOTHER_USER, PASSWORD); + + MailboxProbeImpl mailboxProbe = server.getProbe(MailboxProbeImpl.class); + mailboxProbe.createMailbox("#private", JAMES_USER, DefaultMailboxes.INBOX); + mailboxProbe.createMailbox("#private", OTHER_USER, DefaultMailboxes.INBOX); + mailboxProbe.createMailbox("#private", YET_ANOTHER_USER, DefaultMailboxes.INBOX); + + Port smtpPort = server.getProbe(SmtpGuiceProbe.class).getSmtpPort(); + String message = Resources.toString(Resources.getResource("eml/htmlMail.eml"), StandardCharsets.UTF_8); + + // Given a mail sent to two recipients + try (SMTPMessageSender sender = new SMTPMessageSender(Domain.LOCALHOST.asString())) { + sender.connect(JAMES_SERVER_HOST, smtpPort); + sendUniqueMessageToUsers(sender, message, ImmutableList.of(JAMES_USER, OTHER_USER, YET_ANOTHER_USER)); + } + CALMLY_AWAIT.untilAsserted(() -> assertThat(server.getProbe(SpoolerProbe.class).processingFinished()).isTrue()); + + // When the mails are received + try (TestIMAPClient reader = new TestIMAPClient()) { + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(JAMES_USER, PASSWORD) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(OTHER_USER, PASSWORD_OTHER) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(YET_ANOTHER_USER, PASSWORD) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + } + + // Then the blobs are deduplicated + // (1 mail transiting in the mail queue that is not deleted + // and mailbox messages with headers and body) + // = 1 header (mailqueue) + 1 header (mailbox message) + one body + // = 3 blobs + CassandraProbe probe = server.getProbe(CassandraProbe.class); + ClusterConfiguration cassandraConfiguration = probe.getConfiguration(); + try (Cluster cluster = ClusterFactory.create(cassandraConfiguration, CassandraConsistenciesConfiguration.DEFAULT)) { + try (Session session = cluster.connect(probe.getMainKeyspaceConfiguration().getKeyspace())) { + assertThat(session.execute(select().from(BlobTables.DefaultBucketBlobTable.TABLE_NAME))) + .hasSize(3); + } + } + } +} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreImmutableTest.java new file mode 100644 index 0000000..a4b40b7 --- /dev/null +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreImmutableTest.java @@ -0,0 +1,45 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james; + +import org.apache.james.jmap.draft.JmapJamesServerContract; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.TestJMAPServerModule; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.junit.jupiter.api.extension.RegisterExtension; + +public class WithCassandraPassThroughBlobStoreImmutableTest implements JmapJamesServerContract, JamesServerContract { + @RegisterExtension + static JamesServerExtension jamesServerExtension = new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir -> + CassandraRabbitMQJamesConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.cassandra() + .passthrough()) + .searchConfiguration(SearchConfiguration.elasticSearch()) + .build()) + .server(configuration -> CassandraRabbitMQJamesServerMain.createServer(configuration) + .overrideWith(new TestJMAPServerModule())) + .extension(new DockerElasticSearchExtension()) + .extension(new CassandraExtension()) + .extension(new RabbitMQExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_CLASS) + .build(); +} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreMutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreMutableTest.java new file mode 100644 index 0000000..1dd7ce0 --- /dev/null +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraPassThroughBlobStoreMutableTest.java @@ -0,0 +1,127 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james; + +import static com.datastax.driver.core.querybuilder.QueryBuilder.select; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; + +import org.apache.james.backends.cassandra.init.ClusterFactory; +import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration; +import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration; +import org.apache.james.blob.cassandra.BlobTables; +import org.apache.james.core.Domain; +import org.apache.james.jmap.draft.JmapJamesServerContract; +import org.apache.james.mailbox.DefaultMailboxes; +import org.apache.james.modules.MailboxProbeImpl; +import org.apache.james.modules.RabbitMQExtension; +import org.apache.james.modules.TestJMAPServerModule; +import org.apache.james.modules.blobstore.BlobStoreConfiguration; +import org.apache.james.modules.protocols.ImapGuiceProbe; +import org.apache.james.modules.protocols.SmtpGuiceProbe; +import org.apache.james.server.CassandraProbe; +import org.apache.james.util.Port; +import org.apache.james.utils.DataProbeImpl; +import org.apache.james.utils.SMTPMessageSender; +import org.apache.james.utils.SpoolerProbe; +import org.apache.james.utils.TestIMAPClient; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; + +public class WithCassandraPassThroughBlobStoreMutableTest implements MailsShouldBeWellReceived { + private static final String JAMES_SERVER_HOST = "127.0.0.1"; + private static final String YET_ANOTHER_USER = "yet-another-user@" + DOMAIN; + + @RegisterExtension + static JamesServerExtension jamesServerExtension = new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir -> + CassandraRabbitMQJamesConfiguration.builder() + .workingDirectory(tmpDir) + .configurationFromClasspath() + .blobStore(BlobStoreConfiguration.cassandra() + .passthrough()) + .searchConfiguration(SearchConfiguration.elasticSearch()) + .build()) + .server(configuration -> CassandraRabbitMQJamesServerMain.createServer(configuration) + .overrideWith(new TestJMAPServerModule())) + .extension(new DockerElasticSearchExtension()) + .extension(new CassandraExtension()) + .extension(new RabbitMQExtension()) + .lifeCycle(JamesServerExtension.Lifecycle.PER_TEST) + .build(); + + @Test + void blobsShouldBeDuplicated(GuiceJamesServer server) throws Exception { + server.getProbe(DataProbeImpl.class).fluent() + .addDomain(DOMAIN) + .addUser(JAMES_USER, PASSWORD) + .addUser(OTHER_USER, PASSWORD_OTHER) + .addUser(YET_ANOTHER_USER, PASSWORD); + + MailboxProbeImpl mailboxProbe = server.getProbe(MailboxProbeImpl.class); + mailboxProbe.createMailbox("#private", JAMES_USER, DefaultMailboxes.INBOX); + mailboxProbe.createMailbox("#private", OTHER_USER, DefaultMailboxes.INBOX); + mailboxProbe.createMailbox("#private", YET_ANOTHER_USER, DefaultMailboxes.INBOX); + + Port smtpPort = server.getProbe(SmtpGuiceProbe.class).getSmtpPort(); + String message = Resources.toString(Resources.getResource("eml/htmlMail.eml"), StandardCharsets.UTF_8); + + // Given a mail sent to two recipients + try (SMTPMessageSender sender = new SMTPMessageSender(Domain.LOCALHOST.asString())) { + sender.connect(JAMES_SERVER_HOST, smtpPort); + sendUniqueMessageToUsers(sender, message, ImmutableList.of(JAMES_USER, OTHER_USER, YET_ANOTHER_USER)); + } + CALMLY_AWAIT.untilAsserted(() -> assertThat(server.getProbe(SpoolerProbe.class).processingFinished()).isTrue()); + + // When the mails are received + try (TestIMAPClient reader = new TestIMAPClient()) { + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(JAMES_USER, PASSWORD) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(OTHER_USER, PASSWORD_OTHER) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(YET_ANOTHER_USER, PASSWORD) + .select(TestIMAPClient.INBOX) + .awaitMessageCount(CALMLY_AWAIT, 1); + } + + // Then the blobs are deduplicated + // (1 mail transiting in the mail queue that is deleted + // and 3 mailbox messages with headers and body) = 6 blobs + CassandraProbe probe = server.getProbe(CassandraProbe.class); + ClusterConfiguration cassandraConfiguration = probe.getConfiguration(); + try (Cluster cluster = ClusterFactory.create(cassandraConfiguration, CassandraConsistenciesConfiguration.DEFAULT)) { + try (Session session = cluster.connect(probe.getMainKeyspaceConfiguration().getKeyspace())) { + assertThat(session.execute(select().from(BlobTables.DefaultBucketBlobTable.TABLE_NAME))) + .hasSize(6); + } + } + } +} diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3ImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3ImmutableTest.java index 207f3cc..0a6db6f 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3ImmutableTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3ImmutableTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftImmutableTest.java index f46a3c3..8d651f4 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftImmutableTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftImmutableTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3ImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3ImmutableTest.java index c4172b7..d733079 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3ImmutableTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3ImmutableTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftImmutableTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftImmutableTest.java index aeb59d8..f0cf7ac 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftImmutableTest.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftImmutableTest.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/DockerRabbitMQRule.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/DockerRabbitMQRule.java index 7834db7..08889b8 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/DockerRabbitMQRule.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/DockerRabbitMQRule.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james.modules; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/RabbitMQExtension.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/RabbitMQExtension.java index ac1c489..2da7a41 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/RabbitMQExtension.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/RabbitMQExtension.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james.modules; diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/TestRabbitMQModule.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/TestRabbitMQModule.java index 47012aa..0a100d8 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/TestRabbitMQModule.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/modules/TestRabbitMQModule.java @@ -1,21 +1,21 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ package org.apache.james.modules; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
