This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit a5cd87d87ce0d8ef83939078a768ac388bdbd489 Author: Benoit TELLIER <[email protected]> AuthorDate: Wed Nov 1 20:38:05 2023 +0100 JAMES-3954 Implement RFC-9394 PARTIAL for IMAP FETCH --- .../apache/james/mpt/imapmailbox/suite/Fetch.java | 7 +++ .../apache/james/imap/scripts/FetchPartial.test | 69 ++++++++++++++++++++++ .../apache/james/imap/api/message/FetchData.java | 19 +++++- .../james/imap/api/process/SelectedMailbox.java | 3 + .../imap/decode/parser/FetchCommandParser.java | 4 ++ .../imap/processor/base/SelectedMailboxImpl.java | 5 ++ .../james/imap/processor/base/UidMsnConverter.java | 13 ++++ .../james/imap/processor/fetch/FetchProcessor.java | 56 ++++++++++-------- 8 files changed, 150 insertions(+), 26 deletions(-) diff --git a/mpt/impl/imap-mailbox/core/src/main/java/org/apache/james/mpt/imapmailbox/suite/Fetch.java b/mpt/impl/imap-mailbox/core/src/main/java/org/apache/james/mpt/imapmailbox/suite/Fetch.java index ca160afaa3..6bbc2da361 100644 --- a/mpt/impl/imap-mailbox/core/src/main/java/org/apache/james/mpt/imapmailbox/suite/Fetch.java +++ b/mpt/impl/imap-mailbox/core/src/main/java/org/apache/james/mpt/imapmailbox/suite/Fetch.java @@ -53,6 +53,13 @@ public abstract class Fetch implements ImapTestConstants { .run("FetchEnvelope"); } + @Test + public void testFetchPartial() throws Exception { + simpleScriptedTestProtocol + .withLocale(Locale.US) + .run("FetchPartial"); + } + @Test public void testFetchEnvelopeIT() throws Exception { simpleScriptedTestProtocol diff --git a/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/FetchPartial.test b/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/FetchPartial.test new file mode 100644 index 0000000000..4f0847cfc0 --- /dev/null +++ b/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/FetchPartial.test @@ -0,0 +1,69 @@ +################################################################ +# Licensed to the Apache Software Foundation (ASF) under one # +# or more contributor license agreements. See the NOTICE file # +# distributed with this work for additional information # +# regarding copyright ownership. The ASF licenses this file # +# to you under the Apache License, Version 2.0 (the # +# "License"); you may not use this file except in compliance # +# with the License. You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, # +# software distributed under the License is distributed on an # +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # +# KIND, either express or implied. See the License for the # +# specific language governing permissions and limitations # +# under the License. # +################################################################ + +TIMER start append + +C: A007 APPEND selected {596+} +C: Date: Mon, 10 Feb 1994 21:52:25 -0800 (PST) +C: From: [email protected] +C: Sender: [email protected], Boss <[email protected]> +C: Reply-to: Bin <[email protected]>, Thin Air <[email protected]> +C: Subject: Re: Test 05 +C: To: Fred Foobar <[email protected]>, Sue <[email protected]> +C: Cc: Moo <[email protected]>, Hugh <[email protected]> +C: Bcc: Secret <[email protected]>, Audit <[email protected]> +C: Message-Id: <[email protected]> +C: In-reply-to: <[email protected]> +C: MIME-Version: 1.0 +C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII +C: +C: Works! +C: +S: \* 5 EXISTS +S: \* 5 RECENT +S: A007 OK (\[.+\] )?APPEND completed. + +C: A008 APPEND selected {596+} +C: Date: Mon, 10 Feb 1994 21:52:25 -0800 (PST) +C: From: [email protected] +C: Sender: [email protected], Boss <[email protected]> +C: Reply-to: Bin <[email protected]>, Thin Air <[email protected]> +C: Subject: Re: Test 05 +C: To: group: [email protected], Mary Smithhhhhhhhhhhhh <[email protected]> +C: Cc: Moo <[email protected]>, Hugh <[email protected]> +C: Bcc: Secret <[email protected]>, Audit <[email protected]> +C: Message-Id: <[email protected]> +C: In-reply-to: <[email protected]> +C: MIME-Version: 1.0 +C: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII +C: +C: Works! +C: +S: \* 6 EXISTS +S: \* 6 RECENT +S: A008 OK (\[.+\] )?APPEND completed. + +TIMER print append + +TIMER start fetch + +C: f1 FETCH 2:5 (UID) (PARTIAL 2:3) +S: \* 3 FETCH \(UID 3\) +S: \* 4 FETCH \(UID 4\) +S: f1 OK FETCH completed. diff --git a/protocols/imap/src/main/java/org/apache/james/imap/api/message/FetchData.java b/protocols/imap/src/main/java/org/apache/james/imap/api/message/FetchData.java index 5fb6e496a8..555ecef7ec 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/api/message/FetchData.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/api/message/FetchData.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.Objects; +import java.util.Optional; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; @@ -42,6 +43,7 @@ public class FetchData { private boolean setSeen = false; private long changedSince = -1; private boolean vanished; + private Optional<PartialRange> partialRange = Optional.empty(); public Builder fetch(Item item) { itemToFetch.add(item); @@ -62,6 +64,11 @@ public class FetchData { return fetch(Item.MODSEQ); } + public Builder partial(PartialRange partialRange) { + this.partialRange = Optional.of(partialRange); + return this; + } + /** * Set to true if the VANISHED FETCH modifier was used as stated in <code>QRESYNC</code> extension */ @@ -89,7 +96,7 @@ public class FetchData { } public FetchData build() { - return new FetchData(itemToFetch, bodyElements.build(), setSeen, changedSince, vanished); + return new FetchData(itemToFetch, bodyElements.build(), partialRange, setSeen, changedSince, vanished); } } @@ -115,13 +122,15 @@ public class FetchData { private final EnumSet<Item> itemToFetch; private final ImmutableSet<BodyFetchElement> bodyElements; + private final Optional<PartialRange> partialRange; private final boolean setSeen; private final long changedSince; private final boolean vanished; - private FetchData(EnumSet<Item> itemToFetch, ImmutableSet<BodyFetchElement> bodyElements, boolean setSeen, long changedSince, boolean vanished) { + private FetchData(EnumSet<Item> itemToFetch, ImmutableSet<BodyFetchElement> bodyElements, Optional<PartialRange> partialRange, boolean setSeen, long changedSince, boolean vanished) { this.itemToFetch = itemToFetch; this.bodyElements = bodyElements; + this.partialRange = partialRange; this.setSeen = setSeen; this.changedSince = changedSince; this.vanished = vanished; @@ -142,7 +151,11 @@ public class FetchData { public long getChangedSince() { return changedSince; } - + + public Optional<PartialRange> getPartialRange() { + return partialRange; + } + /** * Return true if the VANISHED FETCH modifier was used as stated in <code>QRESYNC<code> extension */ diff --git a/protocols/imap/src/main/java/org/apache/james/imap/api/process/SelectedMailbox.java b/protocols/imap/src/main/java/org/apache/james/imap/api/process/SelectedMailbox.java index efd430034d..3eeb7b662c 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/api/process/SelectedMailbox.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/api/process/SelectedMailbox.java @@ -20,6 +20,7 @@ package org.apache.james.imap.api.process; import java.util.Collection; +import java.util.List; import java.util.Optional; import javax.mail.Flags; @@ -173,6 +174,8 @@ public interface SelectedMailbox { * empty */ Optional<MessageUid> getLastUid(); + + List<MessageUid> allUids(); /** * Return all applicable Flags for the selected mailbox diff --git a/protocols/imap/src/main/java/org/apache/james/imap/decode/parser/FetchCommandParser.java b/protocols/imap/src/main/java/org/apache/james/imap/decode/parser/FetchCommandParser.java index 10dad694d6..2ee60d9811 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/decode/parser/FetchCommandParser.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/decode/parser/FetchCommandParser.java @@ -114,6 +114,10 @@ public class FetchCommandParser extends AbstractUidCommandParser { request.consumeWord(StringMatcherCharacterValidator.ignoreCase(CHANGEDSINCE)); fetch.changedSince(request.number(true)); return true; + case 'P': + request.consumeWord(StringMatcherCharacterValidator.ignoreCase("PARTIAL")); + fetch.partial(request.parsePartialRange()); + return true; case 'V': // Check for the VANISHED option which is part of QRESYNC request.consumeWord(StringMatcherCharacterValidator.ignoreCase(VANISHED), true); diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/base/SelectedMailboxImpl.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/base/SelectedMailboxImpl.java index c488994b46..53d598a1d1 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/processor/base/SelectedMailboxImpl.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/base/SelectedMailboxImpl.java @@ -190,6 +190,11 @@ public class SelectedMailboxImpl implements SelectedMailbox, EventListener.React return uidMsnConverter.getLastUid(); } + @Override + public List<MessageUid> allUids() { + return uidMsnConverter.allUids(); + } + @Override public Mono<Void> deselect() { return Mono.from( diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/base/UidMsnConverter.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/base/UidMsnConverter.java index 981a85ba27..b4cc4bbf61 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/processor/base/UidMsnConverter.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/base/UidMsnConverter.java @@ -27,6 +27,7 @@ import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.NullableMessageSequenceNumber; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import it.unimi.dsi.fastutil.ints.IntAVLTreeSet; import it.unimi.dsi.fastutil.ints.IntArrayList; @@ -168,6 +169,18 @@ public class UidMsnConverter { return getUid(getLastMsn()); } + public synchronized List<MessageUid> allUids() { + if (usesInts) { + return uidsAsInts.intStream() + .mapToObj(MessageUid::of) + .collect(ImmutableList.toImmutableList()); + } else { + return uids.longStream() + .mapToObj(MessageUid::of) + .collect(ImmutableList.toImmutableList()); + } + } + public synchronized Optional<MessageUid> getFirstUid() { return getUid(FIRST_MSN); } diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/fetch/FetchProcessor.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/fetch/FetchProcessor.java index 64f9d3d9c6..65a437324f 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/processor/fetch/FetchProcessor.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/fetch/FetchProcessor.java @@ -49,6 +49,7 @@ import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageManager; import org.apache.james.mailbox.MessageManager.MailboxMetaData; +import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.exception.MessageRangeException; import org.apache.james.mailbox.model.FetchGroup; @@ -62,8 +63,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongList; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -169,24 +173,38 @@ public class FetchProcessor extends AbstractMailboxProcessor<FetchRequest> { FetchResponseBuilder builder = new FetchResponseBuilder(new EnvelopeBuilder()); FetchGroup resultToFetch = FetchDataConverter.getFetchGroup(fetch); - return Flux.fromIterable(ranges) - .concatMap(range -> { - if (fetch.isOnlyFlags()) { - return processMessageRangeForFlags(selected, mailbox, fetch, mailboxSession, responder, builder, range); - } else { + if (fetch.isOnlyFlags()) { + return Flux.fromIterable(consolidate(selected, ranges, fetch)) + .concatMap(range -> Flux.from(mailbox.listMessagesMetadata(range, mailboxSession))) + .filter(ids -> !fetch.contains(Item.MODSEQ) || ids.getModSeq().asLong() > fetch.getChangedSince()) + .concatMap(result -> toResponse(mailbox, fetch, mailboxSession, builder, selected, result)) + .doOnNext(responder::respond) + .then(); + } else { + return Flux.fromIterable(consolidate(selected, ranges, fetch)) + .concatMap(range -> { auditTrail(mailbox, mailboxSession, resultToFetch, range); - return processMessageRange(selected, mailbox, fetch, mailboxSession, responder, builder, resultToFetch, range); - } - }) - .then(); + return Flux.from(mailbox.getMessagesReactive(range, resultToFetch, mailboxSession)) + .filter(ids -> !fetch.contains(Item.MODSEQ) || ids.getModSeq().asLong() > fetch.getChangedSince()) + .concatMap(result -> toResponse(mailbox, fetch, mailboxSession, builder, selected, result)) + .doOnNext(responder::respond) + .then(); + }) + .then(); + } } - private Mono<Void> processMessageRangeForFlags(SelectedMailbox selected, MessageManager mailbox, FetchData fetch, MailboxSession mailboxSession, Responder responder, FetchResponseBuilder builder, MessageRange range) { - return Flux.from(mailbox.listMessagesMetadata(range, mailboxSession)) - .filter(ids -> !fetch.contains(Item.MODSEQ) || ids.getModSeq().asLong() > fetch.getChangedSince()) - .concatMap(result -> toResponse(mailbox, fetch, mailboxSession, builder, selected, result)) - .doOnNext(responder::respond) - .then(); + List<MessageRange> consolidate(SelectedMailbox selected, List<MessageRange> ranges, FetchData fetchData) { + if (fetchData.getPartialRange().isEmpty()) { + return ranges; + } + LongList longs = new LongArrayList(); + selected.allUids() + .stream() + .filter(uid -> ranges.stream().anyMatch(range -> range.includes(uid))) + .forEach(uid -> longs.add(uid.asLong())); + LongList filter = fetchData.getPartialRange().get().filter(longs); + return MessageRange.toRanges(filter.longStream().mapToObj(MessageUid::of).collect(ImmutableList.toImmutableList())); } private Mono<FetchResponse> toResponse(MessageManager mailbox, FetchData fetch, MailboxSession mailboxSession, FetchResponseBuilder builder, SelectedMailbox selected, org.apache.james.mailbox.model.ComposedMessageIdWithMetaData result) { @@ -226,14 +244,6 @@ public class FetchProcessor extends AbstractMailboxProcessor<FetchRequest> { } } - private Mono<Void> processMessageRange(SelectedMailbox selected, MessageManager mailbox, FetchData fetch, MailboxSession mailboxSession, Responder responder, FetchResponseBuilder builder, FetchGroup resultToFetch, MessageRange range) { - return Flux.from(mailbox.getMessagesReactive(range, resultToFetch, mailboxSession)) - .filter(ids -> !fetch.contains(Item.MODSEQ) || ids.getModSeq().asLong() > fetch.getChangedSince()) - .concatMap(result -> toResponse(mailbox, fetch, mailboxSession, builder, selected, result)) - .doOnNext(responder::respond) - .then(); - } - private static void auditTrail(MessageManager mailbox, MailboxSession mailboxSession, FetchGroup resultToFetch, MessageRange range) { if (resultToFetch.equals(FULL_CONTENT)) { AuditTrail.entry() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
