This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 1727a0e62f203991043493a89316cb2f1b269ae1 Author: Rene Cordier <[email protected]> AuthorDate: Mon Jun 8 13:28:30 2020 +0700 JAMES-3202 Add reindexing mode in RunningOptions --- .../apache/james/mailbox/indexer/ReIndexer.java | 81 +++++++++++++++++++++- .../mailbox/tools/indexer/RunningOptionsDTO.java | 18 ++++- ...rorRecoveryIndexationTaskSerializationTest.java | 17 ++++- .../FullReindexingTaskSerializationTest.java | 9 ++- ...ngleMailboxReindexingTaskSerializationTest.java | 9 ++- .../UserReindexingTaskSerializationTest.java | 9 ++- .../routes/ReindexingRunningOptionsParser.java | 11 ++- 7 files changed, 135 insertions(+), 19 deletions(-) diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java index 2a015f7..60c08a4 100644 --- a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java +++ b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java @@ -19,6 +19,9 @@ package org.apache.james.mailbox.indexer; +import java.util.Optional; +import java.util.stream.Stream; + import org.apache.james.core.Username; import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.exception.MailboxException; @@ -29,17 +32,91 @@ import org.apache.james.task.Task; public interface ReIndexer { class RunningOptions { - public static final RunningOptions DEFAULT = new RunningOptions(50); + public static class Builder { + private Optional<Integer> messagesPerSecond; + private Optional<Mode> mode; + + public Builder() { + this.messagesPerSecond = Optional.empty(); + this.mode = Optional.empty(); + } + + public Builder messagesPerSeconds(Optional<Integer> messagesPerSecond) { + this.messagesPerSecond = messagesPerSecond; + return this; + } + + public Builder messagesPerSeconds(int messagesPerSecond) { + return messagesPerSeconds(Optional.of(messagesPerSecond)); + } + + public Builder mode(Optional<Mode> mode) { + this.mode = mode; + return this; + } + + public Builder mode(Mode mode) { + return mode(Optional.of(mode)); + } + + public RunningOptions build() { + return new RunningOptions( + messagesPerSecond.orElse(DEFAULT_MESSAGES_PER_SECONDS), + mode.orElse(DEFAULT_MODE) + ); + } + } + + public static Builder builder() { + return new Builder(); + } + + public enum Mode { + REBUILD_ALL("rebuildAll"), + FIX_OUTDATED("fixOutdated"); + + private final String value; + + Mode(String value) { + this.value = value; + } + + String getValue() { + return value; + } + + static Optional<Mode> fromString(String optionalMode) { + return Stream.of(values()) + .filter(mode -> mode.getValue().equalsIgnoreCase(optionalMode)) + .findFirst(); + } + } + + public static Optional<Mode> parseMode(String optionalMode) { + return Optional.ofNullable(optionalMode) + .flatMap(Mode::fromString); + } + + private static final Mode DEFAULT_MODE = Mode.REBUILD_ALL; + private static final int DEFAULT_MESSAGES_PER_SECONDS = 50; + + public static final RunningOptions DEFAULT = builder().build(); private final int messagesPerSecond; + private final Mode mode; - public RunningOptions(int messagesPerSecond) { + private RunningOptions(int messagesPerSecond, Mode mode) { this.messagesPerSecond = messagesPerSecond; + this.mode = mode; } public int getMessagesPerSecond() { return messagesPerSecond; } + + public Mode getMode() { + return mode; + } } Task reIndex(Username username, RunningOptions runningOptions) throws MailboxException; diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/RunningOptionsDTO.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/RunningOptionsDTO.java index 16f3d5f..8ae6d56 100644 --- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/RunningOptionsDTO.java +++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/RunningOptionsDTO.java @@ -28,22 +28,34 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class RunningOptionsDTO { public static RunningOptionsDTO toDTO(RunningOptions runningOptions) { - return new RunningOptionsDTO(Optional.of(runningOptions.getMessagesPerSecond())); + return new RunningOptionsDTO( + Optional.of(runningOptions.getMessagesPerSecond()), + Optional.of(runningOptions.getMode())); } private final Optional<Integer> messagesPerSecond; + private final Optional<RunningOptions.Mode> mode; @JsonCreator - public RunningOptionsDTO(@JsonProperty("messagesPerSecond") Optional<Integer> messagesPerSecond) { + public RunningOptionsDTO(@JsonProperty("messagesPerSecond") Optional<Integer> messagesPerSecond, + @JsonProperty("mode") Optional<RunningOptions.Mode> mode) { this.messagesPerSecond = messagesPerSecond; + this.mode = mode; } public Optional<Integer> getMessagesPerSecond() { return messagesPerSecond; } + public Optional<RunningOptions.Mode> getMode() { + return mode; + } + public RunningOptions toDomainObject() { - return new RunningOptions(messagesPerSecond.orElse(RunningOptions.DEFAULT.getMessagesPerSecond())); + return RunningOptions.builder() + .messagesPerSeconds(messagesPerSecond) + .mode(mode) + .build(); } } diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ErrorRecoveryIndexationTaskSerializationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ErrorRecoveryIndexationTaskSerializationTest.java index 3ad7d4f..7ef471c 100644 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ErrorRecoveryIndexationTaskSerializationTest.java +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ErrorRecoveryIndexationTaskSerializationTest.java @@ -43,10 +43,11 @@ class ErrorRecoveryIndexationTaskSerializationTest { private final int successfullyReprocessedMailCount = 42; private final int failedReprocessedMailCount = 2; private final String serializedErrorRecoveryReindexingTask = "{\"type\": \"error-recovery-indexation\"," + - " \"previousMessageFailures\" : [{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"previousMailboxFailures\": [\"3\"], \"runningOptions\":{\"messagesPerSecond\":50}}"; + " \"previousMessageFailures\" : [{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"previousMailboxFailures\": [\"3\"], \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"REBUILD_ALL\"}}"; private final String legacySerializedErrorRecoveryReindexingTask = "{\"type\": \"error-recovery-indexation\"," + " \"previousFailures\" : [{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}]}"; - private final String serializedAdditionalInformation = "{\"type\": \"error-recovery-indexation\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"mailboxFailures\":[\"3\"], \"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedAdditionalInformation = "{\"type\": \"error-recovery-indexation\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"mailboxFailures\":[\"3\"], \"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"REBUILD_ALL\"}}"; + private final String serializedAdditionalInformationWithCorrectMode = "{\"type\": \"error-recovery-indexation\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"mailboxFailures\":[\"3\"], \"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"FIX_OUTDATED\"}}"; private final String legacySerializedAdditionalInformation = "{\"type\": \"error-recovery-indexation\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"failures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"timestamp\":\"2018-11-13T12:00:55Z\"}"; private final TestId mailboxId = TestId.of(1L); private final MessageUid messageUid = MessageUid.of(10L); @@ -97,6 +98,18 @@ class ErrorRecoveryIndexationTaskSerializationTest { } @Test + void additionalInformationWithCorrectModeShouldBeSerializable() throws Exception { + RunningOptions runningOptions = RunningOptions.builder() + .mode(RunningOptions.Mode.FIX_OUTDATED) + .build(); + ReprocessingContextInformationForErrorRecoveryIndexationTask details = new ReprocessingContextInformationForErrorRecoveryIndexationTask(successfullyReprocessedMailCount, failedReprocessedMailCount, executionFailures, TIMESTAMP, runningOptions); + JsonSerializationVerifier.dtoModule(ReprocessingContextInformationForErrorRecoveryIndexationTask.module(mailboxIdFactory)) + .bean(details) + .json(serializedAdditionalInformationWithCorrectMode) + .verify(); + } + + @Test void legacyAdditionalInformationShouldBeDeserializable() throws Exception { ReprocessingContextInformationForErrorRecoveryIndexationTask legacyAdditionalInformation = JsonGenericSerializer.forModules(ReprocessingContextInformationDTO.ReprocessingContextInformationForErrorRecoveryIndexationTask.module(new TestId.Factory())) .withoutNestedType() diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/FullReindexingTaskSerializationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/FullReindexingTaskSerializationTest.java index a19d8d1..81ae782 100644 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/FullReindexingTaskSerializationTest.java +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/FullReindexingTaskSerializationTest.java @@ -47,10 +47,10 @@ class FullReindexingTaskSerializationTest { private final int successfullyReprocessedMailCount = 42; private final int failedReprocessedMailCount = 2; - private final String serializedFullReindexingTask = "{\"type\":\"full-reindexing\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedFullReindexingTask = "{\"type\":\"full-reindexing\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"REBUILD_ALL\"}}"; private final String legacySerializedFullReindexingTask = "{\"type\":\"full-reindexing\"}"; - private final String serializedAdditionalInformation = "{\"type\": \"full-reindexing\", \"runningOptions\":{\"messagesPerSecond\":50}, \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"mailboxFailures\": [\"3\"],\"timestamp\":\"2018-11-13T12:00:55Z\"}"; + private final String serializedAdditionalInformation = "{\"type\": \"full-reindexing\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"FIX_OUTDATED\"}, \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"mailboxFailures\": [\"3\"],\"timestamp\":\"2018-11-13T12:00:55Z\"}"; private final String legacySerializedAdditionalInformation = "{\"type\": \"full-reindexing\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"failures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"timestamp\":\"2018-11-13T12:00:55Z\"}"; private ReIndexingExecutionFailures reIndexingExecutionFailures; @@ -88,7 +88,10 @@ class FullReindexingTaskSerializationTest { @Test void additionalInformationShouldBeSerializable() throws Exception { - ReprocessingContextInformationForFullReindexingTask details = new ReprocessingContextInformationForFullReindexingTask(successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, RunningOptions.DEFAULT); + RunningOptions runningOptions = RunningOptions.builder() + .mode(RunningOptions.Mode.FIX_OUTDATED) + .build(); + ReprocessingContextInformationForFullReindexingTask details = new ReprocessingContextInformationForFullReindexingTask(successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, runningOptions); JsonSerializationVerifier.dtoModule(ReprocessingContextInformationForFullReindexingTask.module(new TestId.Factory())) .bean(details) diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/SingleMailboxReindexingTaskSerializationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/SingleMailboxReindexingTaskSerializationTest.java index df5a50d..dc1eaf6 100644 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/SingleMailboxReindexingTaskSerializationTest.java +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/SingleMailboxReindexingTaskSerializationTest.java @@ -40,9 +40,9 @@ class SingleMailboxReindexingTaskSerializationTest { private final int successfullyReprocessedMailCount = 42; private final int failedReprocessedMailCount = 2; - private final String serializedMailboxReindexingTask = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedMailboxReindexingTask = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"REBUILD_ALL\"}}"; private final String legacySerializedMailboxReindexingTask = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\"}"; - private final String serializedAdditionalInformation = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10, 20]}], \"mailboxFailures\": [\"2\"],\"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedAdditionalInformation = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10, 20]}], \"mailboxFailures\": [\"2\"],\"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"FIX_OUTDATED\"}}"; private final String legacySerializedAdditionalInformation = "{\"type\": \"mailbox-reindexing\", \"mailboxId\": \"1\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"failures\":[{\"mailboxId\":\"1\",\"uids\":[10, 20]}], \"timestamp\":\"2018-11-13T12:00:55Z\"}"; private final TestId mailboxId = TestId.of(1L); @@ -86,7 +86,10 @@ class SingleMailboxReindexingTaskSerializationTest { @Test void additionalInformationShouldBeSerializable() throws Exception { - SingleMailboxReindexingTask.AdditionalInformation details = new SingleMailboxReindexingTask.AdditionalInformation(mailboxId, successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, RunningOptions.DEFAULT); + RunningOptions runningOptions = RunningOptions.builder() + .mode(RunningOptions.Mode.FIX_OUTDATED) + .build(); + SingleMailboxReindexingTask.AdditionalInformation details = new SingleMailboxReindexingTask.AdditionalInformation(mailboxId, successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, runningOptions); JsonSerializationVerifier.dtoModule(SingleMailboxReindexingTaskAdditionalInformationDTO.module(new TestId.Factory())) .bean(details) diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/UserReindexingTaskSerializationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/UserReindexingTaskSerializationTest.java index 7dbb40d..d1a281c 100644 --- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/UserReindexingTaskSerializationTest.java +++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/UserReindexingTaskSerializationTest.java @@ -41,9 +41,9 @@ class UserReindexingTaskSerializationTest { private final int successfullyReprocessedMailCount = 42; private final int failedReprocessedMailCount = 2; - private final String serializedUserReindexingTask = "{\"type\": \"user-reindexing\", \"username\": \"[email protected]\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedUserReindexingTask = "{\"type\": \"user-reindexing\", \"username\": \"[email protected]\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"REBUILD_ALL\"}}"; private final String legacySerializedUserReindexingTask = "{\"type\": \"user-reindexing\", \"username\": \"[email protected]\"}"; - private final String serializedAdditionalInformation = "{\"type\": \"user-reindexing\", \"user\": \"[email protected]\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}],\"mailboxFailures\":[\"3\"], \"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50}}"; + private final String serializedAdditionalInformation = "{\"type\": \"user-reindexing\", \"user\": \"[email protected]\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"messageFailures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}],\"mailboxFailures\":[\"3\"], \"timestamp\":\"2018-11-13T12:00:55Z\", \"runningOptions\":{\"messagesPerSecond\":50, \"mode\":\"FIX_OUTDATED\"}}"; private final String legacySerializedAdditionalInformation = "{\"type\": \"user-reindexing\", \"user\": \"[email protected]\", \"successfullyReprocessedMailCount\":42,\"failedReprocessedMailCount\":2,\"failures\":[{\"mailboxId\":\"1\",\"uids\":[10]},{\"mailboxId\":\"2\",\"uids\":[20]}], \"timestamp\":\"2018-11-13T12:00:55Z\"}"; private final TestId mailboxId = TestId.of(1L); private final MessageUid messageUid = MessageUid.of(10L); @@ -88,7 +88,10 @@ class UserReindexingTaskSerializationTest { @Test void additionalInformationShouldBeSerializable() throws Exception { - UserReindexingTask.AdditionalInformation details = new UserReindexingTask.AdditionalInformation(USERNAME, successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, RunningOptions.DEFAULT); + RunningOptions runningOptions = RunningOptions.builder() + .mode(RunningOptions.Mode.FIX_OUTDATED) + .build(); + UserReindexingTask.AdditionalInformation details = new UserReindexingTask.AdditionalInformation(USERNAME, successfullyReprocessedMailCount, failedReprocessedMailCount, reIndexingExecutionFailures, TIMESTAMP, runningOptions); JsonSerializationVerifier.dtoModule(UserReindexingTaskAdditionalInformationDTO.module(new TestId.Factory())) .bean(details) .json(serializedAdditionalInformation) diff --git a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/routes/ReindexingRunningOptionsParser.java b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/routes/ReindexingRunningOptionsParser.java index c3f869e..d5ce613 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/routes/ReindexingRunningOptionsParser.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/routes/ReindexingRunningOptionsParser.java @@ -28,9 +28,10 @@ import spark.Request; public class ReindexingRunningOptionsParser { public static RunningOptions parse(Request request) { - return intQueryParameter(request, "messagesPerSecond") - .map(RunningOptions::new) - .orElse(RunningOptions.DEFAULT); + return RunningOptions.builder() + .messagesPerSeconds(intQueryParameter(request, "messagesPerSecond")) + .mode(modeQueryParameter(request, "mode")) + .build(); } public static Optional<Integer> intQueryParameter(Request request, String queryParameter) { @@ -42,4 +43,8 @@ public class ReindexingRunningOptionsParser { "strictly positive optional integer", queryParameter), e); } } + + public static Optional<RunningOptions.Mode> modeQueryParameter(Request request, String queryParameter) { + return RunningOptions.parseMode(request.queryParams(queryParameter)); + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
