This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 0192d7022f3775039cb6d47d9cd021bfc70f39fe Author: Benoit Tellier <[email protected]> AuthorDate: Sun Apr 12 16:29:02 2020 +0700 JAMES-3148 Test and correct metadata cleanup upon failures Delete the lowest levels of metadata first so that retries eventually clean up everything. --- .../mailbox/cassandra/DeleteMessageListener.java | 22 ++- .../cassandra/CassandraMailboxManagerTest.java | 220 +++++++++++++++++++++ 2 files changed, 239 insertions(+), 3 deletions(-) diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java index 31529e6..81afcd1 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java @@ -102,9 +102,9 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen messageIdDAO.retrieveMessages(mailboxId, MessageRange.all()) .map(ComposedMessageIdWithMetaData::getComposedMessageId) - .concatMap(metadata -> imapUidDAO.delete((CassandraMessageId) metadata.getMessageId(), mailboxId) - .then(messageIdDAO.delete(mailboxId, metadata.getUid())) - .then(handleDeletion((CassandraMessageId) metadata.getMessageId()))) + .concatMap(metadata -> handleDeletion((CassandraMessageId) metadata.getMessageId(), mailboxId) + .then(imapUidDAO.delete((CassandraMessageId) metadata.getMessageId(), mailboxId)) + .then(messageIdDAO.delete(mailboxId, metadata.getUid()))) .then() .block(); } @@ -119,6 +119,15 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen .then(messageDAO.delete(messageId))); } + private Mono<Void> handleDeletion(CassandraMessageId messageId, CassandraId excludedId) { + return Mono.just(messageId) + .filterWhen(id -> isReferenced(id, excludedId)) + .flatMap(id -> readMessage(id) + .flatMap(message -> deleteUnreferencedAttachments(message).thenReturn(message)) + .flatMap(this::deleteAttachmentMessageIds) + .then(messageDAO.delete(messageId))); + } + private Mono<MessageRepresentation> readMessage(CassandraMessageId id) { return messageDAO.retrieveMessage(id, MessageMapper.FetchType.Metadata); } @@ -149,4 +158,11 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen .hasElements() .map(negate()); } + + private Mono<Boolean> isReferenced(CassandraMessageId id, CassandraId excludedId) { + return imapUidDAO.retrieve(id, ALL_MAILBOXES) + .filter(metadata -> !metadata.getComposedMessageId().getMailboxId().equals(excludedId)) + .hasElements() + .map(negate()); + } } diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java index 3f427a3..8d14e71 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java @@ -18,6 +18,7 @@ ****************************************************************/ package org.apache.james.mailbox.cassandra; +import static org.apache.james.backends.cassandra.Scenario.Builder.fail; import static org.mockito.Mockito.mock; import java.util.Collection; @@ -135,6 +136,161 @@ public class CassandraMailboxManagerTest extends MailboxManagerTest<CassandraMai }); } + @Test + void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteAttachmentFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM attachmentV2 WHERE idAsUUID=:idAsUUID;")); + + mailboxManager.deleteMailbox(inbox, session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block()) + .isEmpty(); + + softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block()) + .isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } + + @Test + void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMessageFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM messageV2 WHERE messageId=:messageId;")); + + mailboxManager.deleteMailbox(inbox, session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block()) + .isEmpty(); + + softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block()) + .isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } + + @Test + void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMailboxContextFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM messageIdTable")); + + mailboxManager.deleteMailbox(inbox, session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block()) + .isEmpty(); + + softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block()) + .isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } + + @Test + void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMailboxContextByIdFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM imapUidTable")); + + mailboxManager.deleteMailbox(inbox, session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block()) + .isEmpty(); + + softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block()) + .isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } @Test void deleteShouldUnreferenceMessageMetadata(CassandraCluster cassandraCluster) throws Exception { @@ -165,6 +321,70 @@ public class CassandraMailboxManagerTest extends MailboxManagerTest<CassandraMai } @Test + void deleteShouldUnreferenceMessageMetadataWhenDeleteMessageFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM messageV2 WHERE messageId=:messageId;")); + + inboxManager.delete(ImmutableList.of(composedMessageId.getUid()), session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } + + @Test + void deleteShouldUnreferenceMessageMetadataWhenDeleteAttachmentFails(CassandraCluster cassandraCluster) throws Exception { + ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() + .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); + + AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session)) + .map(Throwing.function(MessageResult::getLoadedAttachments)) + .flatMap(Collection::stream) + .map(MessageAttachment::getAttachmentId) + .findFirst() + .get(); + + cassandraCluster.getConf().registerScenario(fail() + .times(1) + .whenQueryStartsWith("DELETE FROM attachmentV2 WHERE idAsUUID=:idAsUUID;")); + + inboxManager.delete(ImmutableList.of(composedMessageId.getUid()), session); + + SoftAssertions.assertSoftly(softly -> { + CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId(); + + softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata) + .blockOptional()).isEmpty(); + + softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional()) + .isEmpty(); + + softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block()) + .doesNotContain(cassandraMessageId); + }); + } + + @Test void deleteMailboxShouldNotUnreferenceMessageMetadataWhenOtherReference(CassandraCluster cassandraCluster) throws Exception { ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder() .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
