This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push: new bff0c01358 JAMES-3755 Add a classifiedAsSpam parameter to Rspamd routes (#1172) bff0c01358 is described below commit bff0c01358cac99aa6d5cafbdf98e1f16132c808 Author: Benoit TELLIER <btell...@linagora.com> AuthorDate: Mon Sep 12 17:33:31 2022 +0200 JAMES-3755 Add a classifiedAsSpam parameter to Rspamd routes (#1172) This allows for instance reporting messages manually reported by the user to the Spam mailbox with re-reporting messages already classified as Spam by Rspamd, avoiding creating a positive retro-action loop, and only reporting relevant messages. Similar for Ham reporting. --- third-party/rspamd/README.md | 4 + .../james/rspamd/route/FeedMessageRoute.java | 12 +- .../james/rspamd/task/FeedHamToRspamdTask.java | 2 +- .../james/rspamd/task/FeedHamToRspamdTaskDTO.java | 15 +- .../james/rspamd/task/FeedSpamToRspamdTask.java | 2 +- .../james/rspamd/task/FeedSpamToRspamdTaskDTO.java | 15 +- .../rspamd/task/GetMailboxMessagesService.java | 40 +++-- .../apache/james/rspamd/task/RunningOptions.java | 60 ++++++- .../james/rspamd/route/FeedMessageRouteTest.java | 56 ++++++ ...amToRspamdTaskAdditionalInformationDTOTest.java | 20 ++- .../james/rspamd/task/FeedHamToRspamdTaskTest.java | 187 +++++++++++++++++++- ...amToRspamdTaskAdditionalInformationDTOTest.java | 20 ++- .../rspamd/task/FeedSpamToRspamdTaskTest.java | 188 ++++++++++++++++++++- ...edHamClassifiedAsHam.additionalInformation.json | 12 ++ ...dSpamClassifiedAsHam.additionalInformation.json | 12 ++ 15 files changed, 601 insertions(+), 44 deletions(-) diff --git a/third-party/rspamd/README.md b/third-party/rspamd/README.md index 0191434e68..296af9e9be 100644 --- a/third-party/rspamd/README.md +++ b/third-party/rspamd/README.md @@ -111,6 +111,8 @@ all messages are reported. These inputs represent the same duration: `1d`, `1day`, `86400 seconds`, `86400`... - `samplingProbability` (optional): float between 0 and 1, represent the chance to report each given message to Rspamd. By default, all messages are reported. +- `classifiedAsSpam` (optional): Boolean, true to only include messages tagged as Spam by Rspamd, false for only +messages tagged as ham by Rspamd. If omitted all messages are included. Will return the task id. E.g: ``` @@ -157,6 +159,8 @@ This endpoint has the following param: These inputs represent the same duration: `1d`, `1day`, `86400 seconds`, `86400`... - `samplingProbability` (optional): float between 0 and 1, represent the chance to report each given message to Rspamd. By default, all messages are reported. +- `classifiedAsSpam` (optional): Boolean, true to only include messages tagged as Spam by Rspamd, false for only +messages tagged as ham by Rspamd. If omitted all messages are included. Will return the task id. E.g: ``` diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java index ab2e59612f..962b0a1143 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java @@ -99,7 +99,8 @@ public class FeedMessageRoute implements Routes { Optional<Long> periodInSecond = getPeriod(request); int messagesPerSecond = getMessagesPerSecond(request).orElse(RunningOptions.DEFAULT_MESSAGES_PER_SECOND); double samplingProbability = getSamplingProbability(request).orElse(RunningOptions.DEFAULT_SAMPLING_PROBABILITY); - return new RunningOptions(periodInSecond, messagesPerSecond, samplingProbability); + Optional<Boolean> classifiedAsSpam = getClassifiedAsSpam(request); + return new RunningOptions(periodInSecond, messagesPerSecond, samplingProbability, classifiedAsSpam); } private Optional<Long> getPeriod(Request req) { @@ -140,5 +141,14 @@ public class FeedMessageRoute implements Routes { throw new IllegalArgumentException("'samplingProbability' must be numeric"); } } + + private Optional<Boolean> getClassifiedAsSpam(Request req) { + try { + return Optional.ofNullable(req.queryParams("classifiedAsSpam")) + .map(Boolean::parseBoolean); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException("'classifiedAsSpam' must be a boolean (true|false)"); + } + } } diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java index 6f489af400..f0b687e282 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java @@ -237,7 +237,7 @@ public class FeedHamToRspamdTask implements Task { public Result run() { Optional<Date> afterDate = runningOptions.getPeriodInSecond().map(periodInSecond -> Date.from(clock.instant().minusSeconds(periodInSecond))); try { - return messagesService.getHamMessagesOfAllUser(afterDate, runningOptions.getSamplingProbability(), context) + return messagesService.getHamMessagesOfAllUser(afterDate, runningOptions, context) .transform(ReactorUtils.<MessageResult, Result>throttle() .elements(runningOptions.getMessagesPerSecond()) .per(Duration.ofSeconds(1)) diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java index c42baa896e..c4f9734033 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java @@ -49,12 +49,14 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO { rspamdHttpClient, new RunningOptions(Optional.ofNullable(dto.getPeriodInSecond()), dto.getMessagesPerSecond(), - dto.getSamplingProbability()), + dto.getSamplingProbability(), + dto.getClassifiedAsSpam()), clock)) .toDTOConverter((domain, type) -> new FeedHamToRspamdTaskDTO(type, domain.getRunningOptions().getPeriodInSecond().orElse(null), domain.getRunningOptions().getMessagesPerSecond(), - domain.getRunningOptions().getSamplingProbability())) + domain.getRunningOptions().getSamplingProbability(), + domain.getRunningOptions().getClassifiedAsSpam())) .typeName(FeedHamToRspamdTask.TASK_TYPE.asString()) .withFactory(TaskDTOModule::new); } @@ -64,15 +66,18 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO { private final Long periodInSecond; private final int messagesPerSecond; private final double samplingProbability; + private final Optional<Boolean> classifiedAsSpam; public FeedHamToRspamdTaskDTO(@JsonProperty("type") String type, @JsonProperty("periodInSecond") Long periodInSecond, @JsonProperty("messagesPerSecond") int messagesPerSecond, - @JsonProperty("samplingProbability") double samplingProbability) { + @JsonProperty("samplingProbability") double samplingProbability, + @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) { this.type = type; this.periodInSecond = periodInSecond; this.messagesPerSecond = messagesPerSecond; this.samplingProbability = samplingProbability; + this.classifiedAsSpam = classifiedAsSpam; } @Override @@ -80,6 +85,10 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO { return type; } + public Optional<Boolean> getClassifiedAsSpam() { + return classifiedAsSpam; + } + public Long getPeriodInSecond() { return periodInSecond; } diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java index 9a33b7440c..aae57cb5f5 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java @@ -238,7 +238,7 @@ public class FeedSpamToRspamdTask implements Task { public Result run() { Optional<Date> afterDate = runningOptions.getPeriodInSecond().map(periodInSecond -> Date.from(clock.instant().minusSeconds(periodInSecond))); try { - return messagesService.getMailboxMessagesOfAllUser(SPAM_MAILBOX_NAME, afterDate, runningOptions.getSamplingProbability(), context) + return messagesService.getMailboxMessagesOfAllUser(SPAM_MAILBOX_NAME, afterDate, runningOptions, context) .transform(ReactorUtils.<MessageResult, Task.Result>throttle() .elements(runningOptions.getMessagesPerSecond()) .per(Duration.ofSeconds(1)) diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java index 29ce342a07..7ddd635e7e 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java @@ -49,13 +49,15 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO { rspamdHttpClient, new RunningOptions(Optional.ofNullable(dto.getPeriodInSecond()), dto.getMessagesPerSecond(), - dto.getSamplingProbability()), + dto.getSamplingProbability(), + dto.getClassifiedAsSpam()), clock)) .toDTOConverter((domain, type) -> new FeedSpamToRspamdTaskDTO( type, domain.getRunningOptions().getPeriodInSecond().orElse(null), domain.getRunningOptions().getMessagesPerSecond(), - domain.getRunningOptions().getSamplingProbability())) + domain.getRunningOptions().getSamplingProbability(), + domain.getRunningOptions().getClassifiedAsSpam())) .typeName(FeedSpamToRspamdTask.TASK_TYPE.asString()) .withFactory(TaskDTOModule::new); } @@ -64,16 +66,19 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO { private final Long periodInSecond; private final int messagesPerSecond; private final double samplingProbability; + private final Optional<Boolean> classifiedAsSpam; public FeedSpamToRspamdTaskDTO(@JsonProperty("type") String type, @JsonProperty("periodInSecond") Long periodInSecond, @JsonProperty("messagesPerSecond") int messagesPerSecond, - @JsonProperty("samplingProbability") double samplingProbability) { + @JsonProperty("samplingProbability") double samplingProbability, + @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) { this.type = type; this.periodInSecond = periodInSecond; this.messagesPerSecond = messagesPerSecond; this.samplingProbability = samplingProbability; + this.classifiedAsSpam = classifiedAsSpam; } @Override @@ -92,4 +97,8 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO { public double getSamplingProbability() { return samplingProbability; } + + public Optional<Boolean> getClassifiedAsSpam() { + return classifiedAsSpam; + } } \ No newline at end of file diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java index 95ba0550c8..1e00b58434 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java @@ -30,9 +30,11 @@ import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageIdManager; import org.apache.james.mailbox.MessageManager; import org.apache.james.mailbox.model.FetchGroup; +import org.apache.james.mailbox.model.MailboxMetaData; import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.mailbox.model.MessageRange; import org.apache.james.mailbox.model.MessageResult; +import org.apache.james.mailbox.model.search.MailboxQuery; import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.mail.MessageMapper; import org.apache.james.mailbox.store.mail.model.Message; @@ -62,22 +64,24 @@ public class GetMailboxMessagesService { this.messageIdManager = messageIdManager; } - public Flux<MessageResult> getMailboxMessagesOfAllUser(String mailboxName, Optional<Date> afterDate, double samplingProbability, + public Flux<MessageResult> getMailboxMessagesOfAllUser(String mailboxName, Optional<Date> afterDate, RunningOptions runningOptions, FeedSpamToRspamdTask.Context context) throws UsersRepositoryException { return Iterators.toFlux(userRepository.list()) - .flatMap(username -> getMailboxMessagesOfAUser(username, mailboxName, afterDate, samplingProbability, context), ReactorUtils.DEFAULT_CONCURRENCY); + .flatMap(username -> getMailboxMessagesOfAUser(username, mailboxName, afterDate, runningOptions, context), ReactorUtils.DEFAULT_CONCURRENCY); } - public Flux<MessageResult> getHamMessagesOfAllUser(Optional<Date> afterDate, double samplingProbability, + public Flux<MessageResult> getHamMessagesOfAllUser(Optional<Date> afterDate, RunningOptions runningOptions, FeedHamToRspamdTask.Context context) throws UsersRepositoryException { return Iterators.toFlux(userRepository.list()) - .flatMap(Throwing.function(username -> Flux.fromIterable(mailboxManager.list(mailboxManager.createSystemSession(username))) - .filter(this::hamMailboxesPredicate) - .flatMap(mailboxPath -> getMailboxMessagesOfAUser(username, mailboxPath, afterDate, samplingProbability, context), 2)), ReactorUtils.DEFAULT_CONCURRENCY); + .flatMap(Throwing.function(username -> + Flux.from(mailboxManager.search(MailboxQuery.privateMailboxesBuilder(mailboxManager.createSystemSession(username)).build(), + mailboxManager.createSystemSession(username))) + .filter(mbxMetadata -> hamMailboxesPredicate(mbxMetadata.getPath())) + .flatMap(mbxMetadata -> getMailboxMessagesOfAUser(username, mbxMetadata, afterDate, runningOptions, context), 2)), ReactorUtils.DEFAULT_CONCURRENCY); } private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, String mailboxName, Optional<Date> afterDate, - double samplingProbability, FeedSpamToRspamdTask.Context context) { + RunningOptions runningOptions, FeedSpamToRspamdTask.Context context) { MailboxSession mailboxSession = mailboxManager.createSystemSession(username); return Mono.from(mailboxManager.getMailboxReactive(MailboxPath.forUser(username, mailboxName), mailboxSession)) @@ -85,32 +89,34 @@ public class GetMailboxMessagesService { .flatMapMany(Throwing.function(mailbox -> mapperFactory.getMessageMapper(mailboxSession).findInMailboxReactive(mailbox, MessageRange.all(), MessageMapper.FetchType.METADATA, UNLIMITED))) .doOnNext(mailboxMessageMetaData -> context.incrementSpamMessageCount()) .filter(mailboxMessageMetaData -> afterDate.map(date -> mailboxMessageMetaData.getInternalDate().after(date)).orElse(true)) - .filter(message -> randomBooleanWithProbability(samplingProbability)) + .filter(message -> randomBooleanWithProbability(runningOptions)) .map(Message::getMessageId) .collectList() - .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession)); + .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession)) + .filter(runningOptions.correspondingClassificationFilter()); } - private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, MailboxPath mailboxPath, Optional<Date> afterDate, - double samplingProbability, FeedHamToRspamdTask.Context context) { + private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, MailboxMetaData mailboxMetaData, Optional<Date> afterDate, + RunningOptions runningOptions, FeedHamToRspamdTask.Context context) { MailboxSession mailboxSession = mailboxManager.createSystemSession(username); - return Mono.from(mailboxManager.getMailboxReactive(mailboxPath, mailboxSession)) + return Mono.from(mailboxManager.getMailboxReactive(mailboxMetaData.getId(), mailboxSession)) .map(Throwing.function(MessageManager::getMailboxEntity)) .flatMapMany(Throwing.function(mailbox -> mapperFactory.getMessageMapper(mailboxSession).findInMailboxReactive(mailbox, MessageRange.all(), MessageMapper.FetchType.METADATA, UNLIMITED))) .doOnNext(mailboxMessageMetaData -> context.incrementHamMessageCount()) .filter(mailboxMessageMetaData -> afterDate.map(date -> mailboxMessageMetaData.getInternalDate().after(date)).orElse(true)) - .filter(message -> randomBooleanWithProbability(samplingProbability)) + .filter(message -> randomBooleanWithProbability(runningOptions)) .map(Message::getMessageId) .collectList() - .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession)); + .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession)) + .filter(runningOptions.correspondingClassificationFilter()); } - public static boolean randomBooleanWithProbability(double probability) { - if (probability == 1.0) { + public static boolean randomBooleanWithProbability(RunningOptions runningOptions) { + if (runningOptions.getSamplingProbability() == 1.0) { return true; } - return Math.random() < probability; + return Math.random() < runningOptions.getSamplingProbability(); } private boolean hamMailboxesPredicate(MailboxPath mailboxPath) { diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java index bc354c9511..e808c922ee 100644 --- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java +++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java @@ -20,25 +20,70 @@ package org.apache.james.rspamd.task; import java.util.Optional; +import java.util.function.Predicate; +import org.apache.james.mailbox.exception.MailboxException; +import org.apache.james.mailbox.model.MessageResult; +import org.apache.james.rspamd.RspamdScanner; +import org.apache.james.util.streams.Iterators; + +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; public class RunningOptions { + interface ClassificationFilter extends Predicate<MessageResult> { + ClassificationFilter ALL = any -> true; + ClassificationFilter CLASSIFIED_AS_HAM = new HeaderBasedPredicate("NO"); + ClassificationFilter CLASSIFIED_AS_SPAM = new HeaderBasedPredicate("YES"); + + class HeaderBasedPredicate implements ClassificationFilter { + private final String value; + + public HeaderBasedPredicate(String value) { + this.value = value; + } + + @Override + public boolean test(MessageResult messageResult) { + try { + return Iterators.toStream(messageResult.getHeaders().headers()) + .filter(header -> header.getName().equalsIgnoreCase(RspamdScanner.FLAG_MAIL.asString())) + .findFirst() + .map(header -> header.getValue().equalsIgnoreCase(value)) + // Message was not classified by Rspamd, include it + .orElse(true); + } catch (MailboxException e) { + throw new RuntimeException(e); + } + } + } + } + + + public static final Optional<Long> DEFAULT_PERIOD = Optional.empty(); public static final int DEFAULT_MESSAGES_PER_SECOND = 10; public static final double DEFAULT_SAMPLING_PROBABILITY = 1; - public static final RunningOptions DEFAULT = new RunningOptions(DEFAULT_PERIOD, DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + public static final Optional<Boolean> ALL_MESSAGES = Optional.empty(); + public static final RunningOptions DEFAULT = new RunningOptions(DEFAULT_PERIOD, DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); private final Optional<Long> periodInSecond; private final int messagesPerSecond; private final double samplingProbability; + private final Optional<Boolean> classifiedAsSpam; public RunningOptions(@JsonProperty("periodInSecond") Optional<Long> periodInSecond, @JsonProperty("messagesPerSecond") int messagesPerSecond, - @JsonProperty("samplingProbability") double samplingProbability) { + @JsonProperty("samplingProbability") double samplingProbability, + @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) { this.periodInSecond = periodInSecond; this.messagesPerSecond = messagesPerSecond; this.samplingProbability = samplingProbability; + this.classifiedAsSpam = classifiedAsSpam; + } + + public Optional<Boolean> getClassifiedAsSpam() { + return classifiedAsSpam; } public Optional<Long> getPeriodInSecond() { @@ -52,4 +97,15 @@ public class RunningOptions { public double getSamplingProbability() { return samplingProbability; } + + @JsonIgnore + public ClassificationFilter correspondingClassificationFilter() { + return classifiedAsSpam.map(result -> { + if (result) { + return ClassificationFilter.CLASSIFIED_AS_SPAM; + } else { + return ClassificationFilter.CLASSIFIED_AS_HAM; + } + }).orElse(ClassificationFilter.ALL); + } } diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java index cc45f95d75..c4bd01fb4a 100644 --- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java +++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java @@ -181,6 +181,34 @@ public class FeedMessageRouteTest { .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY)); } + @Test + void taskShouldDisplayClassifiedAsSpamRunningOption() throws MailboxException { + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW)); + appendMessage(ALICE_SPAM_MAILBOX, Date.from(NOW)); + + String taskId = given() + .queryParam("action", "reportSpam") + .queryParam("classifiedAsSpam", "false") + .post() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("additionalInformation.type", is(FeedSpamToRspamdTask.TASK_TYPE.asString())) + .body("additionalInformation.spamMessageCount", is(2)) + .body("additionalInformation.reportedSpamMessageCount", is(2)) + .body("additionalInformation.errorCount", is(0)) + .body("additionalInformation.runningOptions.classifiedAsSpam", is(false)) + .body("additionalInformation.runningOptions.messagesPerSecond", is(RunningOptions.DEFAULT_MESSAGES_PER_SECOND)) + .body("additionalInformation.runningOptions.periodInSecond", is(nullValue())) + .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY)); + } + @Test void taskShouldReportOnlyMailInPeriod() throws MailboxException { appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); @@ -429,6 +457,34 @@ public class FeedMessageRouteTest { .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY)); } + @Test + void taskShouldDisplayClassifiedAsSpamRunningOption() throws MailboxException { + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW)); + appendMessage(ALICE_INBOX_MAILBOX, Date.from(NOW)); + + String taskId = given() + .queryParam("action", "reportHam") + .queryParam("classifiedAsSpam", "true") + .post() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("additionalInformation.type", is(FeedHamToRspamdTask.TASK_TYPE.asString())) + .body("additionalInformation.hamMessageCount", is(2)) + .body("additionalInformation.reportedHamMessageCount", is(2)) + .body("additionalInformation.errorCount", is(0)) + .body("additionalInformation.runningOptions.classifiedAsSpam", is(true)) + .body("additionalInformation.runningOptions.messagesPerSecond", is(RunningOptions.DEFAULT_MESSAGES_PER_SECOND)) + .body("additionalInformation.runningOptions.periodInSecond", is(nullValue())) + .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY)); + } + @Test void taskShouldReportOnlyMailInPeriod() throws MailboxException { appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java index 2705eca987..d9fd376b15 100644 --- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java +++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java @@ -40,6 +40,23 @@ class FeedHamToRspamdTaskAdditionalInformationDTOTest { .verify(); } + @Test + void shouldMatchJsonSerializationContractWhenClassifiedAsHam() throws Exception { + JsonSerializationVerifier.dtoModule(FeedHamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE) + .bean(new FeedHamToRspamdTask.AdditionalInformation( + Instant.parse("2007-12-03T10:15:30.00Z"), + 4, + 2, + 1, + new RunningOptions( + Optional.empty(), + RunningOptions.DEFAULT_MESSAGES_PER_SECOND, + RunningOptions.DEFAULT_SAMPLING_PROBABILITY, + Optional.of(false)))) + .json(ClassLoaderUtils.getSystemResourceAsString("json/feedHamClassifiedAsHam.additionalInformation.json")) + .verify(); + } + @Test void shouldMatchJsonSerializationContractWhenNonEmptyPeriod() throws Exception { JsonSerializationVerifier.dtoModule(FeedHamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE) @@ -51,7 +68,8 @@ class FeedHamToRspamdTaskAdditionalInformationDTOTest { new RunningOptions( Optional.of(3600L), RunningOptions.DEFAULT_MESSAGES_PER_SECOND, - RunningOptions.DEFAULT_SAMPLING_PROBABILITY))) + RunningOptions.DEFAULT_SAMPLING_PROBABILITY, + Optional.empty()))) .json(ClassLoaderUtils.getSystemResourceAsString("json/feedHamNonEmptyPeriod.additionalInformation.json")) .verify(); } diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java index 65a187e9dc..0d9a2d8dd2 100644 --- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java +++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java @@ -21,6 +21,7 @@ package org.apache.james.rspamd.task; import static org.apache.james.rspamd.DockerRspamd.PASSWORD; import static org.apache.james.rspamd.task.FeedSpamToRspamdTaskTest.BOB_SPAM_MAILBOX; +import static org.apache.james.rspamd.task.RunningOptions.ALL_MESSAGES; import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_MESSAGES_PER_SECOND; import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_SAMPLING_PROBABILITY; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -144,7 +145,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskShouldReportHamMessageInPeriod() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND))); @@ -163,7 +164,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskShouldNotReportHamMessageNotInPeriod() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); @@ -182,7 +183,7 @@ public class FeedHamToRspamdTaskTest { @Test void mixedInternalDateCase() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); @@ -202,7 +203,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskWithSamplingProbabilityIsZeroShouldReportNonHamMessage() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0); + DEFAULT_MESSAGES_PER_SECOND, 0, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -238,7 +239,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskWithVeryLowSamplingProbabilityShouldReportNotAllHamMessages() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.01); + DEFAULT_MESSAGES_PER_SECOND, 0.01, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -257,7 +258,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskWithVeryHighSamplingProbabilityShouldReportMoreThanZeroMessage() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.99); + DEFAULT_MESSAGES_PER_SECOND, 0.99, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -276,7 +277,7 @@ public class FeedHamToRspamdTaskTest { @Test void taskWithAverageSamplingProbabilityShouldReportSomeMessages() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.5); + DEFAULT_MESSAGES_PER_SECOND, 0.5, ALL_MESSAGES); task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -292,6 +293,168 @@ public class FeedHamToRspamdTaskTest { }); } + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportHamWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isZero(); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportSpamWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportHamWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportSpamWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportHamWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportSpamWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedHamMessageCount()).isZero(); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + @Test void shouldNotReportMessagesInTrashAndSpamMailboxes() throws MailboxException { appendHamMessage(BOB_TRASH_MAILBOX, Date.from(NOW)); @@ -351,4 +514,14 @@ public class FeedHamToRspamdTaskTest { true, new Flags()); } + + private void appendMessage(MailboxPath mailboxPath, Date internalDate, String header) throws MailboxException { + MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser()); + mailboxManager.getMailbox(mailboxPath, session) + .appendMessage(new ByteArrayInputStream((header + "\r\n\r\n" + String.format("random content %4.3f", Math.random())).getBytes()), + internalDate, + session, + true, + new Flags()); + } } diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java index 8471affa3a..70ba6ce805 100644 --- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java +++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java @@ -43,6 +43,23 @@ class FeedSpamToRspamdTaskAdditionalInformationDTOTest { .verify(); } + @Test + void shouldMatchJsonSerializationContractWhenClassifiedAsHam() throws Exception { + JsonSerializationVerifier.dtoModule(FeedSpamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE) + .bean(new FeedSpamToRspamdTask.AdditionalInformation( + Instant.parse("2007-12-03T10:15:30.00Z"), + 4, + 2, + 1, + new RunningOptions( + Optional.empty(), + RunningOptions.DEFAULT_MESSAGES_PER_SECOND, + RunningOptions.DEFAULT_SAMPLING_PROBABILITY, + Optional.of(false)))) + .json(ClassLoaderUtils.getSystemResourceAsString("json/feedSpamClassifiedAsHam.additionalInformation.json")) + .verify(); + } + @Test void shouldMatchJsonSerializationContractWhenNonEmptyPeriod() throws Exception { JsonSerializationVerifier.dtoModule(FeedSpamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE) @@ -54,7 +71,8 @@ class FeedSpamToRspamdTaskAdditionalInformationDTOTest { new RunningOptions( Optional.of(3600L), DEFAULT_MESSAGES_PER_SECOND, - DEFAULT_SAMPLING_PROBABILITY))) + DEFAULT_SAMPLING_PROBABILITY, + Optional.empty()))) .json(ClassLoaderUtils.getSystemResourceAsString("json/feedSpamNonEmptyPeriod.additionalInformation.json")) .verify(); } diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java index 9f800d73bf..3bd9abd943 100644 --- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java +++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java @@ -21,6 +21,7 @@ package org.apache.james.rspamd.task; import static org.apache.james.rspamd.DockerRspamd.PASSWORD; import static org.apache.james.rspamd.task.FeedSpamToRspamdTask.SPAM_MAILBOX_NAME; +import static org.apache.james.rspamd.task.RunningOptions.ALL_MESSAGES; import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_MESSAGES_PER_SECOND; import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_SAMPLING_PROBABILITY; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -137,7 +138,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskShouldReportSpamMessageInPeriod() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND))); @@ -156,7 +157,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskShouldNotReportSpamMessageNotInPeriod() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); @@ -175,7 +176,7 @@ public class FeedSpamToRspamdTaskTest { @Test void mixedInternalDateCase() throws MailboxException { RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND), - DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY); + DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND))); @@ -195,7 +196,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskWithSamplingProbabilityIsZeroShouldReportNonSpamMessage() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0); + DEFAULT_MESSAGES_PER_SECOND, 0, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -231,7 +232,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskWithVeryLowSamplingProbabilityShouldReportNotAllSpamMessages() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.01); + DEFAULT_MESSAGES_PER_SECOND, 0.01, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -250,7 +251,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskWithVeryHighSamplingProbabilityShouldReportMoreThanZeroMessage() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.99); + DEFAULT_MESSAGES_PER_SECOND, 0.99, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -269,7 +270,7 @@ public class FeedSpamToRspamdTaskTest { @Test void taskWithAverageSamplingProbabilityShouldReportSomeMessages() { RunningOptions runningOptions = new RunningOptions(Optional.empty(), - DEFAULT_MESSAGES_PER_SECOND, 0.5); + DEFAULT_MESSAGES_PER_SECOND, 0.5, ALL_MESSAGES); task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); IntStream.range(0, 10) @@ -285,6 +286,169 @@ public class FeedSpamToRspamdTaskTest { }); } + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportHamWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isZero(); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportSpamWhenClassifiedAsSpamIsTrue() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportHamWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportSpamWhenClassifiedAsSpamIsOmited() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty()); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportUnclassifiedWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldReportHamWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + @Test + void shouldNotReportSpamWhenClassifiedAsSpamIsFalse() throws Exception { + RunningOptions runningOptions = new RunningOptions(Optional.empty(), + DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false)); + task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock); + + appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES"); + + Task.Result result = task.run(); + + SoftAssertions.assertSoftly(softly -> { + assertThat(result).isEqualTo(Task.Result.COMPLETED); + assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1); + assertThat(task.snapshot().getReportedSpamMessageCount()).isZero(); + assertThat(task.snapshot().getErrorCount()).isZero(); + }); + } + + private void appendSpamMessage(MailboxPath mailboxPath, Date internalDate) throws MailboxException { MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser()); mailboxManager.getMailbox(mailboxPath, session) @@ -294,4 +458,14 @@ public class FeedSpamToRspamdTaskTest { true, new Flags()); } + + private void appendMessage(MailboxPath mailboxPath, Date internalDate, String header) throws MailboxException { + MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser()); + mailboxManager.getMailbox(mailboxPath, session) + .appendMessage(new ByteArrayInputStream((header + "\r\n\r\n" + String.format("random content %4.3f", Math.random())).getBytes()), + internalDate, + session, + true, + new Flags()); + } } diff --git a/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json b/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json new file mode 100644 index 0000000000..11f189f3be --- /dev/null +++ b/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json @@ -0,0 +1,12 @@ +{ + "errorCount": 1, + "reportedHamMessageCount": 2, + "runningOptions": { + "messagesPerSecond": 10, + "samplingProbability": 1.0, + "classifiedAsSpam": false + }, + "hamMessageCount": 4, + "timestamp": "2007-12-03T10:15:30Z", + "type": "FeedHamToRspamdTask" +} \ No newline at end of file diff --git a/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json b/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json new file mode 100644 index 0000000000..61260e5f27 --- /dev/null +++ b/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json @@ -0,0 +1,12 @@ +{ + "errorCount": 1, + "reportedSpamMessageCount": 2, + "runningOptions": { + "messagesPerSecond": 10, + "samplingProbability": 1.0, + "classifiedAsSpam": false + }, + "spamMessageCount": 4, + "timestamp": "2007-12-03T10:15:30Z", + "type": "FeedSpamToRspamdTask" +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org For additional commands, e-mail: notifications-h...@james.apache.org