This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 7fcc86b199b52cb09c2311a222f4e79b402af67b Author: Benoit TELLIER <[email protected]> AuthorDate: Wed Sep 30 09:44:25 2026 +0200 [ENHANCEMENT] SolveMessageInconsistencies: offer to cleanup mailbox entry without backing content --- .../operate/webadmin/admin-mailboxes-extend.adoc | 16 +- .../operate/webadmin/admin-messages-extend.adoc | 16 +- .../cassandra/mail/task/RunningOptionsDTO.java | 15 +- .../task/SolveMessageInconsistenciesService.java | 159 ++++++++++++++++-- .../mail/task/SolveMessageInconsistenciesTask.java | 12 +- ...nconsistenciesTaskAdditionalInformationDTO.java | 9 ++ .../SolveMessageInconsistenciesServiceTest.java | 178 +++++++++++++++++++++ ...essageInconsistenciesTaskSerializationTest.java | 5 +- ...ssageInconsistencies.additionalInformation.json | 4 +- .../json/solveMessageInconsistencies.task.json | 3 +- ...itMQWebAdminServerIntegrationImmutableTest.java | 2 + .../webadmin/routes/RunningOptionsParser.java | 9 +- ...lveMessageInconsistenciesRequestToTaskTest.java | 35 ++++ 13 files changed, 437 insertions(+), 26 deletions(-) diff --git a/docs/modules/servers/pages/distributed/operate/webadmin/admin-mailboxes-extend.adoc b/docs/modules/servers/pages/distributed/operate/webadmin/admin-mailboxes-extend.adoc index df7194bd54..208ca5445c 100644 --- a/docs/modules/servers/pages/distributed/operate/webadmin/admin-mailboxes-extend.adoc +++ b/docs/modules/servers/pages/distributed/operate/webadmin/admin-mailboxes-extend.adoc @@ -187,6 +187,18 @@ Default is 100. This optional parameter must have a strictly positive integer as a value and be passed as query parameter. +An admin can also request the removal of the entries whose message +content is missing: + +* `cleanupEntriesWithoutContent` when `true`, the content (`messageV3` +table) of every `imapUidTable` entry is checked, and entries without +content are removed from both `imapUidTable` and `messageIdTable`. +Entries of messages created less than a day ago are never removed. +Default is `false`, in which case such entries are only reported when +detected as orphan. + +This optional parameter must be passed as query parameter. + An admin can also specify the reindexing mode it wants to use when running the task: @@ -227,8 +239,10 @@ The scheduled task will have the following type "addedMessageIdEntries": 1, "updatedMessageIdEntries": 0, "removedMessageIdEntries": 1, + "removedImapUidEntries": 0, "runningOptions":{ "messagesPerSecond": 200, + "cleanupEntriesWithoutContent": false, "mode":"REBUILD_ALL" }, "fixedInconsistencies": [ @@ -325,4 +339,4 @@ The scheduled task will have the following type "errors": ["551f0580-82fb-11ea-970e-f9c83d4cf8c2"], "targetFlag": "RECENT" } -.... \ No newline at end of file +.... diff --git a/docs/modules/servers/pages/distributed/operate/webadmin/admin-messages-extend.adoc b/docs/modules/servers/pages/distributed/operate/webadmin/admin-messages-extend.adoc index 0d5df6519c..e484b19150 100644 --- a/docs/modules/servers/pages/distributed/operate/webadmin/admin-messages-extend.adoc +++ b/docs/modules/servers/pages/distributed/operate/webadmin/admin-messages-extend.adoc @@ -48,6 +48,18 @@ Default is 100. This optional parameter must have a strictly positive integer as a value and be passed as query parameter. +An admin can also request the removal of the entries whose message +content is missing: + +* `cleanupEntriesWithoutContent` when `true`, the content (`messageV3` +table) of every `imapUidTable` entry is checked, and entries without +content are removed from both `imapUidTable` and `messageIdTable`. +Entries of messages created less than a day ago are never removed. +Default is `false`, in which case such entries are only reported when +detected as orphan. + +This optional parameter must be passed as query parameter. + An admin can also specify the reindexing mode it wants to use when running the task: @@ -88,8 +100,10 @@ The scheduled task will have the following type "addedMessageIdEntries": 1, "updatedMessageIdEntries": 0, "removedMessageIdEntries": 1, + "removedImapUidEntries": 0, "runningOptions":{ "messagesPerSecond": 200, + "cleanupEntriesWithoutContent": false, "mode":"REBUILD_ALL" }, "fixedInconsistencies": [ @@ -122,4 +136,4 @@ task will eventually fix all issues. This task could be run safely online and can be scheduled on a recurring basis outside of peak traffic by an admin to ensure Cassandra message -consistency. \ No newline at end of file +consistency. diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/RunningOptionsDTO.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/RunningOptionsDTO.java index 96c21fd43c..a35d05195b 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/RunningOptionsDTO.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/RunningOptionsDTO.java @@ -28,21 +28,30 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class RunningOptionsDTO { public static RunningOptionsDTO asDTO(SolveMessageInconsistenciesService.RunningOptions domainObject) { - return new RunningOptionsDTO(Optional.of(domainObject.getMessagesPerSecond())); + return new RunningOptionsDTO(Optional.of(domainObject.getMessagesPerSecond()), + Optional.of(domainObject.isCleanupEntriesWithoutContent())); } private final Optional<Integer> messagesPerSecond; + private final Optional<Boolean> cleanupEntriesWithoutContent; @JsonCreator - public RunningOptionsDTO(@JsonProperty("messagesPerSecond") Optional<Integer> messagesPerSecond) { + public RunningOptionsDTO(@JsonProperty("messagesPerSecond") Optional<Integer> messagesPerSecond, + @JsonProperty("cleanupEntriesWithoutContent") Optional<Boolean> cleanupEntriesWithoutContent) { this.messagesPerSecond = messagesPerSecond; + this.cleanupEntriesWithoutContent = cleanupEntriesWithoutContent; } public Optional<Integer> getMessagesPerSecond() { return messagesPerSecond; } + public Optional<Boolean> getCleanupEntriesWithoutContent() { + return cleanupEntriesWithoutContent; + } + public RunningOptions asDomainObject() { - return new RunningOptions(messagesPerSecond.orElse(RunningOptions.DEFAULT.getMessagesPerSecond())); + return new RunningOptions(messagesPerSecond.orElse(RunningOptions.DEFAULT.getMessagesPerSecond()), + cleanupEntriesWithoutContent.orElse(RunningOptions.DEFAULT_CLEANUP_ENTRIES_WITHOUT_CONTENT)); } } diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesService.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesService.java index 4a5fc1cacf..43aa4086aa 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesService.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesService.java @@ -23,10 +23,13 @@ import static org.apache.james.backends.cassandra.init.configuration.JamesExecut import static org.apache.james.backends.cassandra.init.configuration.JamesExecutionProfiles.ConsistencyChoice.WEAK; import static org.apache.james.util.ReactorUtils.publishIfPresent; +import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.Collection; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; @@ -129,11 +132,64 @@ public class SolveMessageInconsistenciesService { @Override public Mono<Task.Result> fix(Context context, CassandraMessageIdToImapUidDAO imapUidDAO, CassandraMessageIdDAO messageIdDAO) { context.addErrors(message.getComposedMessageId().getComposedMessageId()); - LOGGER.warn("Skipping orphan message in ImapUid as its content is missing in MessageV3: {}", message.getComposedMessageId()); + LOGGER.warn("Skipping message in ImapUid as its content is missing in MessageV3: {}", message.getComposedMessageId()); return Mono.just(Task.Result.PARTIAL); } } + /** + * Same as {@link ImapUidEntryWithoutContent} but the admin explicitly asked for such entries to be removed. + * + * The entry is removed from both ImapUid and MessageId. Mailbox counters, quotas and search indexes are not + * updated and need to be recomputed. + * + * The content is checked again right before removal, so that removals are only based on up to date reads. + */ + private static class RemovableImapUidEntryWithoutContent implements Inconsistency { + private final CassandraMessageMetadata message; + private final Mono<Boolean> hasContent; + + private RemovableImapUidEntryWithoutContent(CassandraMessageMetadata message, Mono<Boolean> hasContent) { + this.message = message; + this.hasContent = hasContent; + } + + @Override + public Mono<Task.Result> fix(Context context, CassandraMessageIdToImapUidDAO imapUidDAO, CassandraMessageIdDAO messageIdDAO) { + return hasContent + .flatMap(contentFound -> { + if (contentFound) { + LOGGER.warn("Content found in MessageV3 upon re-check, skipping removal of {}", message.getComposedMessageId()); + return Mono.just(Task.Result.COMPLETED); + } + return remove(context, imapUidDAO, messageIdDAO); + }) + .onErrorResume(error -> { + notifyFailure(context, error); + return Mono.just(Task.Result.PARTIAL); + }); + } + + private Mono<Task.Result> remove(Context context, CassandraMessageIdToImapUidDAO imapUidDAO, CassandraMessageIdDAO messageIdDAO) { + ComposedMessageId id = message.getComposedMessageId().getComposedMessageId(); + return imapUidDAO.delete((CassandraMessageId) id.getMessageId(), (CassandraId) id.getMailboxId()) + .then(messageIdDAO.delete((CassandraId) id.getMailboxId(), id.getUid())) + .then(Mono.fromRunnable(() -> notifySuccess(context))) + .thenReturn(Task.Result.COMPLETED); + } + + private void notifyFailure(Context context, Throwable error) { + context.addErrors(message.getComposedMessageId().getComposedMessageId()); + LOGGER.error("Failed to remove message without content in MessageV3: {}", message.getComposedMessageId(), error); + } + + private void notifySuccess(Context context) { + LOGGER.warn("Removed message without content in MessageV3: {}", message.getComposedMessageId()); + context.incrementRemovedImapUidEntries(); + context.addFixedInconsistency(message.getComposedMessageId().getComposedMessageId()); + } + } + private static class OutdatedMessageIdEntry implements Inconsistency { private final CassandraMessageMetadata messageFromMessageId; private final CassandraMessageMetadata messageFromImapUid; @@ -208,19 +264,30 @@ public class SolveMessageInconsistenciesService { public static class RunningOptions { + public static final boolean DEFAULT_CLEANUP_ENTRIES_WITHOUT_CONTENT = false; public static final RunningOptions DEFAULT = new RunningOptions(100); private final int messagesPerSecond; + private final boolean cleanupEntriesWithoutContent; public RunningOptions(int messagesPerSecond) { + this(messagesPerSecond, DEFAULT_CLEANUP_ENTRIES_WITHOUT_CONTENT); + } + + public RunningOptions(int messagesPerSecond, boolean cleanupEntriesWithoutContent) { Preconditions.checkArgument(messagesPerSecond > 0, "'messagesPerSecond' must be strictly positive"); this.messagesPerSecond = messagesPerSecond; + this.cleanupEntriesWithoutContent = cleanupEntriesWithoutContent; } public int getMessagesPerSecond() { return this.messagesPerSecond; } + + public boolean isCleanupEntriesWithoutContent() { + return cleanupEntriesWithoutContent; + } } public static class Context { @@ -235,6 +302,7 @@ public class SolveMessageInconsistenciesService { private Optional<Long> addedMessageIdEntries; private Optional<Long> updatedMessageIdEntries; private Optional<Long> removedMessageIdEntries; + private Optional<Long> removedImapUidEntries; private ImmutableList.Builder<ComposedMessageId> fixedInconsistencies; private ImmutableList.Builder<ComposedMessageId> errors; @@ -244,6 +312,7 @@ public class SolveMessageInconsistenciesService { addedMessageIdEntries = Optional.empty(); updatedMessageIdEntries = Optional.empty(); removedMessageIdEntries = Optional.empty(); + removedImapUidEntries = Optional.empty(); fixedInconsistencies = ImmutableList.builder(); errors = ImmutableList.builder(); } @@ -273,6 +342,11 @@ public class SolveMessageInconsistenciesService { return this; } + public Builder removedImapUidEntries(long count) { + removedImapUidEntries = Optional.of(count); + return this; + } + public Builder addFixedInconsistencies(ComposedMessageId composedMessageId) { fixedInconsistencies.add(composedMessageId); return this; @@ -290,6 +364,7 @@ public class SolveMessageInconsistenciesService { addedMessageIdEntries.orElse(0L), updatedMessageIdEntries.orElse(0L), removedMessageIdEntries.orElse(0L), + removedImapUidEntries.orElse(0L), fixedInconsistencies.build(), errors.build()); } @@ -300,12 +375,13 @@ public class SolveMessageInconsistenciesService { private final long addedMessageIdEntries; private final long updatedMessageIdEntries; private final long removedMessageIdEntries; + private final long removedImapUidEntries; private final ImmutableList<ComposedMessageId> fixedInconsistencies; private final ImmutableList<ComposedMessageId> errors; private Snapshot(long processedImapUidEntries, long processedMessageIdEntries, long addedMessageIdEntries, long updatedMessageIdEntries, - long removedMessageIdEntries, + long removedMessageIdEntries, long removedImapUidEntries, ImmutableList<ComposedMessageId> fixedInconsistencies, ImmutableList<ComposedMessageId> errors) { this.processedImapUidEntries = processedImapUidEntries; @@ -313,6 +389,7 @@ public class SolveMessageInconsistenciesService { this.addedMessageIdEntries = addedMessageIdEntries; this.updatedMessageIdEntries = updatedMessageIdEntries; this.removedMessageIdEntries = removedMessageIdEntries; + this.removedImapUidEntries = removedImapUidEntries; this.fixedInconsistencies = fixedInconsistencies; this.errors = errors; } @@ -337,6 +414,10 @@ public class SolveMessageInconsistenciesService { return removedMessageIdEntries; } + public long getRemovedImapUidEntries() { + return removedImapUidEntries; + } + public ImmutableList<ComposedMessageId> getFixedInconsistencies() { return fixedInconsistencies; } @@ -355,6 +436,7 @@ public class SolveMessageInconsistenciesService { && Objects.equals(this.addedMessageIdEntries, snapshot.addedMessageIdEntries) && Objects.equals(this.updatedMessageIdEntries, snapshot.updatedMessageIdEntries) && Objects.equals(this.removedMessageIdEntries, snapshot.removedMessageIdEntries) + && Objects.equals(this.removedImapUidEntries, snapshot.removedImapUidEntries) && Objects.equals(this.errors, snapshot.errors) && Objects.equals(this.fixedInconsistencies, snapshot.fixedInconsistencies); } @@ -363,7 +445,7 @@ public class SolveMessageInconsistenciesService { @Override public final int hashCode() { - return Objects.hash(processedImapUidEntries, processedMessageIdEntries, addedMessageIdEntries, updatedMessageIdEntries, removedMessageIdEntries, fixedInconsistencies, errors); + return Objects.hash(processedImapUidEntries, processedMessageIdEntries, addedMessageIdEntries, updatedMessageIdEntries, removedMessageIdEntries, removedImapUidEntries, fixedInconsistencies, errors); } @Override @@ -374,6 +456,7 @@ public class SolveMessageInconsistenciesService { .add("addedMessageIdEntries", addedMessageIdEntries) .add("updatedMessageIdEntries", updatedMessageIdEntries) .add("removedMessageIdEntries", removedMessageIdEntries) + .add("removedImapUidEntries", removedImapUidEntries) .add("fixedInconsistencies", fixedInconsistencies) .add("errors", errors) .toString(); @@ -385,21 +468,23 @@ public class SolveMessageInconsistenciesService { private final AtomicLong addedMessageIdEntries; private final AtomicLong updatedMessageIdEntries; private final AtomicLong removedMessageIdEntries; + private final AtomicLong removedImapUidEntries; private final ConcurrentLinkedDeque<ComposedMessageId> fixedInconsistencies; private final ConcurrentLinkedDeque<ComposedMessageId> errors; Context() { - this(new AtomicLong(), new AtomicLong(), new AtomicLong(), new AtomicLong(), new AtomicLong(), ImmutableList.of(), ImmutableList.of()); + this(new AtomicLong(), new AtomicLong(), new AtomicLong(), new AtomicLong(), new AtomicLong(), new AtomicLong(), ImmutableList.of(), ImmutableList.of()); } private Context(AtomicLong processedImapUidEntries, AtomicLong processedMessageIdEntries, AtomicLong addedMessageIdEntries, - AtomicLong updatedMessageIdEntries, AtomicLong removedMessageIdEntries, + AtomicLong updatedMessageIdEntries, AtomicLong removedMessageIdEntries, AtomicLong removedImapUidEntries, Collection<ComposedMessageId> fixedInconsistencies, Collection<ComposedMessageId> errors) { this.processedImapUidEntries = processedImapUidEntries; this.processedMessageIdEntries = processedMessageIdEntries; this.addedMessageIdEntries = addedMessageIdEntries; this.updatedMessageIdEntries = updatedMessageIdEntries; this.removedMessageIdEntries = removedMessageIdEntries; + this.removedImapUidEntries = removedImapUidEntries; this.fixedInconsistencies = new ConcurrentLinkedDeque<>(fixedInconsistencies); this.errors = new ConcurrentLinkedDeque<>(errors); } @@ -424,6 +509,10 @@ public class SolveMessageInconsistenciesService { removedMessageIdEntries.incrementAndGet(); } + void incrementRemovedImapUidEntries() { + removedImapUidEntries.incrementAndGet(); + } + void addFixedInconsistency(ComposedMessageId messageId) { fixedInconsistencies.add(messageId); } @@ -439,6 +528,7 @@ public class SolveMessageInconsistenciesService { addedMessageIdEntries.get(), updatedMessageIdEntries.get(), removedMessageIdEntries.get(), + removedImapUidEntries.get(), ImmutableList.copyOf(fixedInconsistencies), ImmutableList.copyOf(errors)); } @@ -446,19 +536,29 @@ public class SolveMessageInconsistenciesService { private static final Logger LOGGER = LoggerFactory.getLogger(SolveMessageInconsistenciesService.class); private static final Duration PERIOD = Duration.ofSeconds(1); + // Entries of messages created recently are never removed: their content could be written but not visible yet + private static final Duration CLEANUP_GRACE_PERIOD = Duration.ofDays(1); + private static final long UUID_EPOCH_OFFSET_IN_100NS = 0x01B21DD213814000L; private final CassandraMessageIdToImapUidDAO messageIdToImapUidDAO; private final CassandraMessageIdDAO messageIdDAO; private final CassandraMessageDAOV3 messageDAOV3; private final CassandraConfiguration cassandraConfiguration; + private final Clock clock; @Inject SolveMessageInconsistenciesService(CassandraMessageIdToImapUidDAO messageIdToImapUidDAO, CassandraMessageIdDAO messageIdDAO, CassandraMessageDAOV3 messageDAOV3, CassandraConfiguration cassandraConfiguration) { + this(messageIdToImapUidDAO, messageIdDAO, messageDAOV3, cassandraConfiguration, Clock.systemUTC()); + } + + SolveMessageInconsistenciesService(CassandraMessageIdToImapUidDAO messageIdToImapUidDAO, CassandraMessageIdDAO messageIdDAO, + CassandraMessageDAOV3 messageDAOV3, CassandraConfiguration cassandraConfiguration, Clock clock) { this.messageIdToImapUidDAO = messageIdToImapUidDAO; this.messageIdDAO = messageIdDAO; this.messageDAOV3 = messageDAOV3; this.cassandraConfiguration = cassandraConfiguration; + this.clock = clock; } private ConsistencyChoice chooseReadConsistency() { @@ -480,17 +580,52 @@ public class SolveMessageInconsistenciesService { .transform(ReactorUtils.<CassandraMessageMetadata, Task.Result>throttle() .elements(runningOptions.getMessagesPerSecond()) .per(PERIOD) - .forOperation(metaData -> detectInconsistencyInImapUid(metaData) + .forOperation(metaData -> detectInconsistencyInImapUid(metaData, runningOptions) .doOnNext(any -> context.incrementProcessedImapUidEntries()) .flatMap(inconsistency -> inconsistency.fix(context, messageIdToImapUidDAO, messageIdDAO)))); } - private Mono<Inconsistency> detectInconsistencyInImapUid(CassandraMessageMetadata message) { - return compareWithMessageIdRecord(message) + private Mono<Inconsistency> detectInconsistencyInImapUid(CassandraMessageMetadata message, RunningOptions runningOptions) { + return checkContentIfNeeded(message, runningOptions) + .switchIfEmpty(Mono.defer(() -> compareWithMessageIdRecord(message, runningOptions))) .onErrorResume(error -> Mono.just(new FailedToRetrieveRecord(message))); } - private Mono<Inconsistency> compareWithMessageIdRecord(CassandraMessageMetadata messageFromImapUid) { + // Entries resurrected in both ImapUid and MessageId are consistent with each other: the content is thus checked + // for every entry when a cleanup is requested. + private Mono<Inconsistency> checkContentIfNeeded(CassandraMessageMetadata message, RunningOptions runningOptions) { + if (!runningOptions.isCleanupEntriesWithoutContent()) { + return Mono.empty(); + } + CassandraMessageId messageId = (CassandraMessageId) message.getComposedMessageId().getComposedMessageId().getMessageId(); + return hasContent(messageId) + .filter(hasContent -> !hasContent) + .map(any -> entryWithoutContent(message, runningOptions)); + } + + private Inconsistency entryWithoutContent(CassandraMessageMetadata message, RunningOptions runningOptions) { + CassandraMessageId messageId = (CassandraMessageId) message.getComposedMessageId().getComposedMessageId().getMessageId(); + if (runningOptions.isCleanupEntriesWithoutContent() && isOutsideGracePeriod(messageId)) { + return new RemovableImapUidEntryWithoutContent(message, Mono.defer(() -> hasContent(messageId))); + } + return new ImapUidEntryWithoutContent(message); + } + + private boolean isOutsideGracePeriod(CassandraMessageId messageId) { + return creationInstant(messageId) + .map(creation -> creation.plus(CLEANUP_GRACE_PERIOD).isBefore(clock.instant())) + .orElse(false); + } + + private static Optional<Instant> creationInstant(CassandraMessageId messageId) { + UUID uuid = messageId.get(); + if (uuid.version() != 1) { + return Optional.empty(); + } + return Optional.of(Instant.ofEpochMilli((uuid.timestamp() - UUID_EPOCH_OFFSET_IN_100NS) / 10_000)); + } + + private Mono<Inconsistency> compareWithMessageIdRecord(CassandraMessageMetadata messageFromImapUid, RunningOptions runningOptions) { ComposedMessageId ids = messageFromImapUid.getComposedMessageId().getComposedMessageId(); CassandraId mailboxId = (CassandraId) ids.getMailboxId(); MessageUid uid = ids.getUid(); @@ -505,7 +640,7 @@ public class SolveMessageInconsistenciesService { return detectOutdatedMessageIdEntry(mailboxId, messageId, messageIdRecord); }) .switchIfEmpty( - detectOrphanImapUidEntry(mailboxId, messageId)); + detectOrphanImapUidEntry(mailboxId, messageId, runningOptions)); } private Mono<Inconsistency> detectOutdatedMessageIdEntry(CassandraId mailboxId, CassandraMessageId messageId, CassandraMessageMetadata messageIdRecord) { @@ -516,7 +651,7 @@ public class SolveMessageInconsistenciesService { .switchIfEmpty(Mono.just(NO_INCONSISTENCY)); } - private Mono<Inconsistency> detectOrphanImapUidEntry(CassandraId mailboxId, CassandraMessageId messageId) { + private Mono<Inconsistency> detectOrphanImapUidEntry(CassandraId mailboxId, CassandraMessageId messageId, RunningOptions runningOptions) { return messageIdToImapUidDAO.retrieve(messageId, Optional.of(mailboxId), chooseReadConsistency()) .next() .flatMap(orphanEntry -> hasContent(messageId) @@ -524,7 +659,7 @@ public class SolveMessageInconsistenciesService { if (hasContent) { return new OrphanImapUidEntry(orphanEntry); } - return new ImapUidEntryWithoutContent(orphanEntry); + return entryWithoutContent(orphanEntry, runningOptions); })) .switchIfEmpty(Mono.just(NO_INCONSISTENCY)); } diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTask.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTask.java index 5a5ec7f64d..b619aa8122 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTask.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTask.java @@ -44,12 +44,14 @@ public class SolveMessageInconsistenciesTask implements Task { private final long addedMessageIdEntries; private final long updatedMessageIdEntries; private final long removedMessageIdEntries; + private final long removedImapUidEntries; private final RunningOptions runningOptions; private final ImmutableList<MessageInconsistenciesEntry> fixedInconsistencies; private final ImmutableList<MessageInconsistenciesEntry> errors; public Details(Instant instant, long processedImapUidEntries, long processedMessageIdEntries, - long addedMessageIdEntries, long updatedMessageIdEntries, long removedMessageIdEntries, RunningOptions runningOptions, + long addedMessageIdEntries, long updatedMessageIdEntries, long removedMessageIdEntries, + long removedImapUidEntries, RunningOptions runningOptions, ImmutableList<MessageInconsistenciesEntry> fixedInconsistencies, ImmutableList<MessageInconsistenciesEntry> errors) { this.instant = instant; this.processedImapUidEntries = processedImapUidEntries; @@ -57,6 +59,7 @@ public class SolveMessageInconsistenciesTask implements Task { this.addedMessageIdEntries = addedMessageIdEntries; this.updatedMessageIdEntries = updatedMessageIdEntries; this.removedMessageIdEntries = removedMessageIdEntries; + this.removedImapUidEntries = removedImapUidEntries; this.runningOptions = runningOptions; this.fixedInconsistencies = fixedInconsistencies; this.errors = errors; @@ -87,6 +90,10 @@ public class SolveMessageInconsistenciesTask implements Task { return removedMessageIdEntries; } + public long getRemovedImapUidEntries() { + return removedImapUidEntries; + } + public RunningOptions getRunningOptions() { return runningOptions; } @@ -129,7 +136,8 @@ public class SolveMessageInconsistenciesTask implements Task { public Optional<TaskExecutionDetails.AdditionalInformation> details() { Snapshot snapshot = context.snapshot(); return Optional.of(new Details(Clock.systemUTC().instant(), snapshot.getProcessedImapUidEntries(), snapshot.getProcessedMessageIdEntries(), - snapshot.getAddedMessageIdEntries(), snapshot.getUpdatedMessageIdEntries(), snapshot.getRemovedMessageIdEntries(), runningOptions, + snapshot.getAddedMessageIdEntries(), snapshot.getUpdatedMessageIdEntries(), snapshot.getRemovedMessageIdEntries(), + snapshot.getRemovedImapUidEntries(), runningOptions, snapshot.getFixedInconsistencies().stream() .map(this::toMessageInconsistenciesEntry) .collect(ImmutableList.toImmutableList()), diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskAdditionalInformationDTO.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskAdditionalInformationDTO.java index ca1484c383..d22e2c2e86 100644 --- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskAdditionalInformationDTO.java +++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskAdditionalInformationDTO.java @@ -44,6 +44,7 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements details.getAddedMessageIdEntries(), details.getUpdatedMessageIdEntries(), details.getRemovedMessageIdEntries(), + details.getRemovedImapUidEntries(), Optional.of(RunningOptionsDTO.asDTO(details.getRunningOptions())), details.getFixedInconsistencies(), details.getErrors()); @@ -65,6 +66,7 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements private final long addedMessageIdEntries; private final long updatedMessageIdEntries; private final long removedMessageIdEntries; + private final long removedImapUidEntries; private final Optional<RunningOptionsDTO> runningOptions; private final ImmutableList<MessageInconsistenciesEntry> fixedInconsistencies; private final ImmutableList<MessageInconsistenciesEntry> errors; @@ -75,6 +77,7 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements @JsonProperty("addedMessageIdEntries") long addedMessageIdEntries, @JsonProperty("updatedMessageIdEntries") long updatedMessageIdEntries, @JsonProperty("removedMessageIdEntries")long removedMessageIdEntries, + @JsonProperty("removedImapUidEntries") long removedImapUidEntries, @JsonProperty("runningOptions") Optional<RunningOptionsDTO> runningOptions, @JsonProperty("fixedInconsistencies") ImmutableList<MessageInconsistenciesEntry> fixedInconsistencies, @JsonProperty("errors") ImmutableList<MessageInconsistenciesEntry> errors) { @@ -85,6 +88,7 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements this.addedMessageIdEntries = addedMessageIdEntries; this.updatedMessageIdEntries = updatedMessageIdEntries; this.removedMessageIdEntries = removedMessageIdEntries; + this.removedImapUidEntries = removedImapUidEntries; this.runningOptions = runningOptions; this.fixedInconsistencies = fixedInconsistencies; this.errors = errors; @@ -110,6 +114,10 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements return removedMessageIdEntries; } + public long getRemovedImapUidEntries() { + return removedImapUidEntries; + } + public Optional<RunningOptionsDTO> getRunningOptions() { return runningOptions; } @@ -139,6 +147,7 @@ public class SolveMessageInconsistenciesTaskAdditionalInformationDTO implements addedMessageIdEntries, updatedMessageIdEntries, removedMessageIdEntries, + removedImapUidEntries, runningOptions .map(RunningOptionsDTO::asDomainObject) .orElse(RunningOptions.DEFAULT), diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesServiceTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesServiceTest.java index 2311f55d98..402f587d55 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesServiceTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesServiceTest.java @@ -20,10 +20,14 @@ package org.apache.james.mailbox.cassandra.mail.task; import static org.apache.james.backends.cassandra.Scenario.Builder.awaitOn; +import static org.apache.james.backends.cassandra.Scenario.Builder.executeNormally; import static org.apache.james.backends.cassandra.Scenario.Builder.fail; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Date; import java.util.Optional; @@ -671,6 +675,180 @@ public class SolveMessageInconsistenciesServiceTest { }); } + @Nested + class CleanupEntriesWithoutContentTest { + private static final RunningOptions CLEANUP = new RunningOptions(100, true); + + @Test + void cleanupShouldRemoveEntriesWithoutContent() { + // Resurrected in both tables: ImapUid and MessageId are consistent with each other + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + testee.fixMessageInconsistencies(new Context(), CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .isEmpty(); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .isEmpty(); + }); + } + + @Test + void cleanupShouldReportRemovedEntries() { + Context context = new Context(); + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + Task.Result result = testee.fixMessageInconsistencies(context, CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.COMPLETED); + softly.assertThat(context.snapshot()) + .isEqualTo(Context.Snapshot.builder() + .processedImapUidEntries(1) + .processedMessageIdEntries(1) + .removedImapUidEntries(1) + .addFixedInconsistencies(MESSAGE_3.getComposedMessageId().getComposedMessageId()) + .build()); + }); + } + + @Test + void cleanupShouldRemoveOrphanImapUidEntriesWithoutContent() { + imapUidDAO.insert(MESSAGE_3).block(); + + testee.fixMessageInconsistencies(new Context(), CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .isEmpty(); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .isEmpty(); + }); + } + + @Test + void cleanupShouldNotRemoveEntriesWithContent() { + imapUidDAO.insert(MESSAGE_1).block(); + messageIdDAO.insert(MESSAGE_1).block(); + + Task.Result result = testee.fixMessageInconsistencies(new Context(), CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.COMPLETED); + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_1); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_1); + }); + } + + @Test + void cleanupShouldNotRemoveRecentEntries() { + // One hour after MESSAGE_ID_3 creation: within the grace period + Clock clock = Clock.fixed(Instant.parse("2020-04-14T16:24:15Z"), ZoneOffset.UTC); + testee = new SolveMessageInconsistenciesService(imapUidDAO, messageIdDAO, messageDAOV3, CassandraConfiguration.DEFAULT_CONFIGURATION, clock); + Context context = new Context(); + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + Task.Result result = testee.fixMessageInconsistencies(context, CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.PARTIAL); + softly.assertThat(context.snapshot().getErrors()) + .containsExactly(MESSAGE_3.getComposedMessageId().getComposedMessageId()); + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + }); + } + + @Test + void cleanupShouldNotRemoveEntriesWhoseContentAppearsBeforeRemoval(CassandraCluster cassandra) throws Exception { + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + // The detection read is executed normally, the re-check read right before the removal is blocked + Scenario.Barrier barrier = new Scenario.Barrier(1); + cassandra.getConf() + .registerScenario( + executeNormally() + .times(1) + .whenQueryStartsWith("SELECT * FROM messagev3 WHERE messageid=:messageid"), + awaitOn(barrier) + .thenExecuteNormally() + .times(1) + .whenQueryStartsWith("SELECT * FROM messagev3 WHERE messageid=:messageid")); + + Context context = new Context(); + Mono<Task.Result> task = testee.fixMessageInconsistencies(context, CLEANUP).subscribeOn(Schedulers.boundedElastic()).cache(); + task.subscribe(); + + barrier.awaitCaller(); + saveContent(MESSAGE_ID_3); + barrier.releaseCaller(); + + Task.Result result = task.block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.COMPLETED); + softly.assertThat(context.snapshot().getRemovedImapUidEntries()).isZero(); + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + }); + } + + @Test + void cleanupShouldNotRemoveEntriesWhenContentReCheckFails(CassandraCluster cassandra) { + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + cassandra.getConf() + .registerScenario( + executeNormally() + .times(1) + .whenQueryStartsWith("SELECT * FROM messagev3 WHERE messageid=:messageid"), + fail() + .times(1) + .whenQueryStartsWith("SELECT * FROM messagev3 WHERE messageid=:messageid")); + + Context context = new Context(); + Task.Result result = testee.fixMessageInconsistencies(context, CLEANUP).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.PARTIAL); + softly.assertThat(context.snapshot().getErrors()) + .containsExactly(MESSAGE_3.getComposedMessageId().getComposedMessageId()); + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + }); + } + + @Test + void entriesWithoutContentShouldNotBeRemovedWhenNoCleanup() { + imapUidDAO.insert(MESSAGE_3).block(); + messageIdDAO.insert(MESSAGE_3).block(); + + Task.Result result = testee.fixMessageInconsistencies(new Context(), RunningOptions.DEFAULT).block(); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(result).isEqualTo(Task.Result.COMPLETED); + softly.assertThat(imapUidDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + softly.assertThat(messageIdDAO.retrieveAllMessages().collectList().block()) + .containsExactly(MESSAGE_3); + }); + } + } + @Test void fixMailboxInconsistenciesShouldUpdateContextWhenInconsistentModSeq() { Context context = new Context(); diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskSerializationTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskSerializationTest.java index cdd2f37013..7bda5bba06 100644 --- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskSerializationTest.java +++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/task/SolveMessageInconsistenciesTaskSerializationTest.java @@ -37,7 +37,7 @@ import com.google.common.collect.ImmutableList; public class SolveMessageInconsistenciesTaskSerializationTest { private static final SolveMessageInconsistenciesService SERVICE = mock(SolveMessageInconsistenciesService.class); - private static final SolveMessageInconsistenciesTask TASK = new SolveMessageInconsistenciesTask(SERVICE, new RunningOptions(2)); + private static final SolveMessageInconsistenciesTask TASK = new SolveMessageInconsistenciesTask(SERVICE, new RunningOptions(2, true)); private static final Instant INSTANT = Instant.parse("2007-12-03T10:15:30.00Z"); private static final String MAILBOX_ID = "551f0580-82fb-11ea-970e-f9c83d4cf8c2"; @@ -61,7 +61,7 @@ public class SolveMessageInconsistenciesTaskSerializationTest { .messageId(MESSAGE_ID_3) .messageUid(MESSAGE_UID_3); - private static final Details DETAILS = new SolveMessageInconsistenciesTask.Details(INSTANT, 2, 1, 1, 0, 1, new SolveMessageInconsistenciesService.RunningOptions(2), ImmutableList.of(MESSAGE_1, MESSAGE_2), ImmutableList.of(MESSAGE_3)); + private static final Details DETAILS = new SolveMessageInconsistenciesTask.Details(INSTANT, 2, 1, 1, 0, 1, 1, new SolveMessageInconsistenciesService.RunningOptions(2, true), ImmutableList.of(MESSAGE_1, MESSAGE_2), ImmutableList.of(MESSAGE_3)); @Test void taskShouldBeSerializable() throws Exception { @@ -108,6 +108,7 @@ public class SolveMessageInconsistenciesTaskSerializationTest { 1, 0, 1, + 0, RunningOptions.DEFAULT, ImmutableList.of(MESSAGE_1, MESSAGE_2), ImmutableList.of(MESSAGE_3) diff --git a/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.additionalInformation.json b/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.additionalInformation.json index d6516879f5..92046a2b1c 100644 --- a/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.additionalInformation.json +++ b/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.additionalInformation.json @@ -6,8 +6,10 @@ "addedMessageIdEntries": 1, "updatedMessageIdEntries": 0, "removedMessageIdEntries": 1, + "removedImapUidEntries": 1, "runningOptions":{ - "messagesPerSecond": 2 + "messagesPerSecond": 2, + "cleanupEntriesWithoutContent": true }, "fixedInconsistencies": [ { diff --git a/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.task.json b/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.task.json index 92b1eddd5e..5eb82d4eb1 100644 --- a/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.task.json +++ b/mailbox/cassandra/src/test/resources/json/solveMessageInconsistencies.task.json @@ -1,6 +1,7 @@ { "type":"solve-message-inconsistencies", "runningOptions":{ - "messagesPerSecond":2 + "messagesPerSecond":2, + "cleanupEntriesWithoutContent":true } } \ No newline at end of file diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerIntegrationImmutableTest.java b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerIntegrationImmutableTest.java index a93b003df8..7ede034cf9 100644 --- a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerIntegrationImmutableTest.java +++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerIntegrationImmutableTest.java @@ -118,7 +118,9 @@ class RabbitMQWebAdminServerIntegrationImmutableTest extends WebAdminServerInteg .body("additionalInformation.addedMessageIdEntries", is(0)) .body("additionalInformation.updatedMessageIdEntries", is(0)) .body("additionalInformation.removedMessageIdEntries", is(0)) + .body("additionalInformation.removedImapUidEntries", is(0)) .body("additionalInformation.runningOptions.messagesPerSecond", is(100)) + .body("additionalInformation.runningOptions.cleanupEntriesWithoutContent", is(false)) .body("additionalInformation.fixedInconsistencies", hasSize(0)) .body("additionalInformation.errors", hasSize(0)); } diff --git a/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/routes/RunningOptionsParser.java b/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/routes/RunningOptionsParser.java index 9d689224cb..bb74146519 100644 --- a/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/routes/RunningOptionsParser.java +++ b/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/routes/RunningOptionsParser.java @@ -27,9 +27,12 @@ import spark.Request; public class RunningOptionsParser { public static RunningOptions parse(Request request) { - return intQueryParameter(request, "messagesPerSecond") - .map(RunningOptions::new) - .orElse(RunningOptions.DEFAULT); + int messagesPerSecond = intQueryParameter(request, "messagesPerSecond") + .orElse(RunningOptions.DEFAULT.getMessagesPerSecond()); + boolean cleanupEntriesWithoutContent = Optional.ofNullable(request.queryParams("cleanupEntriesWithoutContent")) + .map(Boolean::parseBoolean) + .orElse(RunningOptions.DEFAULT_CLEANUP_ENTRIES_WITHOUT_CONTENT); + return new RunningOptions(messagesPerSecond, cleanupEntriesWithoutContent); } public static Optional<Integer> intQueryParameter(Request request, String queryParameter) { diff --git a/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/SolveMessageInconsistenciesRequestToTaskTest.java b/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/SolveMessageInconsistenciesRequestToTaskTest.java index cfc34eef28..3c22a34e86 100644 --- a/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/SolveMessageInconsistenciesRequestToTaskTest.java +++ b/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/SolveMessageInconsistenciesRequestToTaskTest.java @@ -226,4 +226,39 @@ class SolveMessageInconsistenciesRequestToTaskTest { .body("type", is("solve-message-inconsistencies")) .body("additionalInformation.runningOptions.messagesPerSecond", is(20)); } + + @Test + void cleanupEntriesWithoutContentShouldBePartOfTaskDetails() { + String taskId = with() + .queryParam("action", "SolveInconsistencies") + .queryParam("cleanupEntriesWithoutContent", "true") + .post() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("taskId", is(taskId)) + .body("additionalInformation.runningOptions.cleanupEntriesWithoutContent", is(true)); + } + + @Test + void cleanupEntriesWithoutContentShouldBeDisabledByDefault() { + String taskId = with() + .queryParam("action", "SolveInconsistencies") + .post() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("taskId", is(taskId)) + .body("additionalInformation.runningOptions.cleanupEntriesWithoutContent", is(false)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
