congbobo184 commented on code in PR #18273:
URL: https://github.com/apache/pulsar/pull/18273#discussion_r1095326157
##########
pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java:
##########
@@ -0,0 +1,147 @@
+package org.apache.pulsar.broker.transaction;
+
+import java.lang.reflect.Field;
+import java.util.LinkedList;
+import java.util.NavigableMap;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.commons.collections4.map.LinkedMap;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import
org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class AbortTxnProcessorTest extends TransactionTestBase {
+
+ private static final String PROCESSOR_TOPIC = "persistent://" + NAMESPACE1
+ "/abortedTxnProcessor";
+ private static final int SEGMENT_SIZE = 5;
+ private PulsarService pulsarService = null;
+
+ @Override
+ @BeforeClass
+ protected void setup() throws Exception {
+ setUpBase(1, 1, PROCESSOR_TOPIC, 0);
+ this.pulsarService = getPulsarServiceList().get(0);
+
this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
+
this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 +
PROCESSOR_TOPIC.length() + 5 * 3);
+ }
+
+ @Override
+ @AfterClass
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ /**
+ * Test api:
+ * 1. putAbortedTxnAndPosition
+ * 2. checkAbortedTransaction
+ * 3. takeAbortedTxnsSnapshot
+ * 4. recoverFromSnapshot
+ * 5. trimExpiredAbortedTxns
+ * @throws Exception
+ */
+ @Test
+ public void testPutAbortedTxnIntoProcessor() throws Exception {
+ PersistentTopic persistentTopic = (PersistentTopic)
pulsarService.getBrokerService()
+ .getTopic(PROCESSOR_TOPIC, false).get().get();
+ AbortedTxnProcessor processor = new
SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic);
+ //1. prepare test data.
+ //1.1 Put 10 aborted txn IDs to persistent two sealed segments.
+ for (int i = 0; i < 10; i++) {
+ TxnID txnID = new TxnID(0, i);
+ PositionImpl position = new PositionImpl(0,i);
+ processor.putAbortedTxnAndPosition(txnID, position);
+ }
+ //1.2 Put 4 aborted txn IDs into the unsealed segment.
+ for (int i = 10; i < 14; i++) {
+ TxnID txnID = new TxnID(0, i);
+ PositionImpl position = new PositionImpl(0,i);
+ processor.putAbortedTxnAndPosition(txnID, position);
+ }
+ //1.3 Verify the common data flow
+ verifyAbortedTxnIDAndSegmentIndex(processor,0,14);
+ //2. Take the latest snapshot and verify recover from snapshot
+ AbortedTxnProcessor newProcessor = new
SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic);
+ PositionImpl maxReadPosition = new PositionImpl(0, 14);
+ //2.1 Avoid update operation being canceled.
+ waitTaskExecuteCompletely(processor);
+ //2.2 take the latest snapshot
+ processor.takeAbortedTxnsSnapshot(maxReadPosition).get();
+ newProcessor.recoverFromSnapshot().get();
+ //Verify the recovery data flow
+ verifyAbortedTxnIDAndSegmentIndex(newProcessor,0,14);
+ //3. Delete the ledgers and then verify the date.
+ Field ledgersField =
ManagedLedgerImpl.class.getDeclaredField("ledgers");
+ ledgersField.setAccessible(true);
+ NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers
=
+ (NavigableMap<Long,
MLDataFormats.ManagedLedgerInfo.LedgerInfo>)
+ ledgersField.get(persistentTopic.getManagedLedger());
+ ledgers.forEach((k, v) -> {
Review Comment:
could we verify one ledger delete, delete one snapshot segment? and add
clear snapshot test.
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java:
##########
@@ -0,0 +1,721 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,2
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.buffer.impl;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Supplier;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl;
+import org.apache.commons.collections4.map.LinkedMap;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.systopic.SystemTopicClient;
+import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment;
+import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.common.naming.SystemTopicNames;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.common.util.FutureUtil;
+
+@Slf4j
+public class SnapshotSegmentAbortedTxnProcessorImpl implements
AbortedTxnProcessor {
+
+ /**
+ * Stored the unsealed aborted transaction IDs Whose size is always less
than the snapshotSegmentCapacity.
+ * It will be persistent as a snapshot segment when its size reach the
configured capacity.
+ */
+ private LinkedList<TxnID> unsealedTxnIds;
+
+ /**
+ * The map is used to clear the aborted transaction IDs persistent in the
expired ledger.
+ * <p>
+ * The key PositionImpl {@link PositionImpl} is the persistent
position of
+ * the latest transaction of a segment.
+ * The value TxnID {@link TxnID} is the latest Transaction ID in a
segment.
+ * </p>
+ *
+ * <p>
+ * If the position is expired, the processor can get the according
latest
+ * transaction ID in this map. And then the processor can clear all the
+ * transaction IDs in the aborts {@link
SnapshotSegmentAbortedTxnProcessorImpl#aborts}
+ * that lower than the transaction ID.
+ * And then the processor can delete the segments persistently
according to
+ * the positions.
+ * </p>
+ */
+ private final LinkedMap<PositionImpl, TxnID> segmentIndex = new
LinkedMap<>();
+
+ /**
+ * This map is used to check whether a transaction is an aborted
transaction.
+ * <p>
+ * The transaction IDs is appended in order, so the processor can
delete expired
+ * transaction IDs according to the latest expired transaction IDs in
segmentIndex
+ * {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}.
+ * </p>
+ */
+ private final LinkedMap<TxnID, TxnID> aborts = new LinkedMap<>();
+ /**
+ * This map stores the indexes of the snapshot segment.
+ * <p>
+ * The key is the persistent position of the marker of the last
transaction in the segment.
+ * The value TransactionBufferSnapshotIndex {@link
TransactionBufferSnapshotIndex} is the
+ * indexes of the snapshot segment.
+ * </p>
+ */
+ private final LinkedMap<PositionImpl, TransactionBufferSnapshotIndex>
indexes = new LinkedMap<>();
+
+ private final PersistentTopic topic;
+
+ private volatile long lastSnapshotTimestamps;
+
+ /**
+ * The number of the aborted transaction IDs in a segment.
+ * This is calculated according to the configured memory size.
+ */
+ private final int snapshotSegmentCapacity;
+ /**
+ * Responsible for executing the persistent tasks.
+ * <p>Including:</p>
+ * <p> Update segment index.</p>
+ * <p> Write snapshot segment.</p>
+ * <p> Delete snapshot segment.</p>
+ * <p> Clear all snapshot segment. </p>
+ */
+ private final PersistentWorker persistentWorker;
+
+ private static final String SNAPSHOT_PREFIX = "multiple-";
+
+ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) {
+ this.topic = topic;
+ this.persistentWorker = new PersistentWorker(topic);
+ /*
+ Calculate the segment capital according to its size configuration.
+ <p>
+ The empty transaction segment size is 5.
+ Adding an empty linkedList, the size increase to 6.
+ Add the topic name the size increase to the 7 +
topic.getName().length().
+ Add the aborted transaction IDs, the size increase to 8 +
+ topic.getName().length() + 3 * aborted transaction ID size.
+ </p>
+ */
+ this.snapshotSegmentCapacity = (topic.getBrokerService().getPulsar()
+ .getConfiguration().getTransactionBufferSnapshotSegmentSize()
- 8 - topic.getName().length()) / 3;
+ this.unsealedTxnIds = new LinkedList<>();
+ }
+
+ @Override
+ public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) {
+ unsealedTxnIds.add(txnID);
+ aborts.put(txnID, txnID);
+ /*
+ The size of lastAbortedTxns reaches the configuration of the size
of snapshot segment.
+ Append a task to persistent the segment with the aborted
transaction IDs and the latest
+ transaction mark persistent position passed by param.
+ */
+ if (unsealedTxnIds.size() >= snapshotSegmentCapacity) {
+ LinkedList<TxnID> abortedSegment = unsealedTxnIds;
+ segmentIndex.put(position, txnID);
+
persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () ->
+ persistentWorker.takeSnapshotSegmentAsync(abortedSegment,
position));
+ this.unsealedTxnIds = new LinkedList<>();
+ }
+ }
+
+ @Override
+ public boolean checkAbortedTransaction(TxnID txnID) {
+ return aborts.containsKey(txnID);
+ }
+
+ /**
+ * Check werther the position in segmentIndex {@link
SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}
+ * is expired. If the position is not exist in the original topic, the
according transaction is an invalid
+ * transaction. And the according segment is invalid, too. The transaction
IDs before the transaction ID
+ * in the aborts are invalid, too.
+ */
+ @Override
+ public void trimExpiredAbortedTxns() {
+ //Checking whether there are some segment expired.
+ List<PositionImpl> positionsNeedToDelete = new ArrayList<>();
+ while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl)
topic.getManagedLedger())
+ .ledgerExists(segmentIndex.firstKey().getLedgerId())) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Topic transaction buffer clear aborted
transactions, maxReadPosition : {}",
+ topic.getName(), segmentIndex.firstKey());
+ }
+ PositionImpl positionNeedToDelete = segmentIndex.firstKey();
+ positionsNeedToDelete.add(positionNeedToDelete);
+
+ TxnID theLatestDeletedTxnID = segmentIndex.remove(0);
+ while (!aborts.firstKey().equals(theLatestDeletedTxnID)) {
+ aborts.remove(0);
+ }
+ aborts.remove(0);
+ }
+ //Batch delete the expired segment
+ if (!positionsNeedToDelete.isEmpty()) {
+
persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment,
+ () ->
persistentWorker.deleteSnapshotSegment(positionsNeedToDelete));
+ }
+ }
+
+ private String buildKey(long sequenceId) {
+ return SNAPSHOT_PREFIX + sequenceId + "-" + this.topic.getName();
+ }
+
+ @Override
+ public CompletableFuture<Void> takeAbortedTxnsSnapshot(PositionImpl
maxReadPosition) {
+ //Store the latest aborted transaction IDs in unsealedTxnIDs and the
according the latest max read position.
+ TransactionBufferSnapshotIndexesMetadata metadata = new
TransactionBufferSnapshotIndexesMetadata(
+ maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(),
+ convertTypeToTxnIDData(unsealedTxnIds));
+ return
persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () ->
persistentWorker
+ .updateSnapshotIndex(metadata,
indexes.values().stream().toList()));
+ }
+
+ @Override
+ public CompletableFuture<PositionImpl> recoverFromSnapshot() {
+ return
topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory()
+ .getTxnBufferSnapshotIndexService()
+
.createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> {
+ PositionImpl startReadCursorPosition = null;
+ TransactionBufferSnapshotIndexes persistentSnapshotIndexes
= null;
+ boolean hasIndex = false;
+ try {
+ /*
+ Read the transaction snapshot segment index.
+ <p>
+ The processor can get the sequence ID, unsealed
transaction IDs,
+ segment index list and max read position in the
snapshot segment index.
+ Then we can traverse the index list to read all
aborted transaction IDs
+ in segments to aborts.
+ </p>
+ */
+ while (reader.hasMoreEvents()) {
+ Message<TransactionBufferSnapshotIndexes> message
= reader.readNextAsync()
+ .get(getSystemClientOperationTimeoutMs(),
TimeUnit.MILLISECONDS);
+ if (topic.getName().equals(message.getKey())) {
+ TransactionBufferSnapshotIndexes
transactionBufferSnapshotIndexes = message.getValue();
+ if (transactionBufferSnapshotIndexes != null) {
+ hasIndex = true;
+ persistentSnapshotIndexes =
transactionBufferSnapshotIndexes;
+ startReadCursorPosition = PositionImpl.get(
+
transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(),
+
transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId());
+ }
+ }
+ }
+ } catch (TimeoutException ex) {
+ Throwable t = FutureUtil.unwrapCompletionException(ex);
+ String errorMessage = String.format("[%s] Transaction
buffer recover fail by read "
+ + "transactionBufferSnapshot timeout!",
topic.getName());
+ log.error(errorMessage, t);
+ return FutureUtil.failedFuture(
+ new
BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t));
+ } catch (Exception ex) {
+ log.error("[{}] Transaction buffer recover fail when
read "
+ + "transactionBufferSnapshot!",
topic.getName(), ex);
+ return FutureUtil.failedFuture(ex);
+ } finally {
+ closeReader(reader);
+ }
+ PositionImpl finalStartReadCursorPosition =
startReadCursorPosition;
+ TransactionBufferSnapshotIndexes
finalPersistentSnapshotIndexes = persistentSnapshotIndexes;
+ if (!hasIndex) {
+ return CompletableFuture.completedFuture(null);
+ } else {
+ this.unsealedTxnIds =
convertTypeToTxnID(persistentSnapshotIndexes
+ .getSnapshot().getAborts());
+ }
+ //Read snapshot segment to recover aborts.
+ ArrayList<CompletableFuture<Void>> completableFutures =
new ArrayList<>();
+ CompletableFuture<Void> openManagedLedgerFuture = new
CompletableFuture<>();
+ AtomicBoolean hasInvalidIndex = new AtomicBoolean(false);
+ AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback
= new AsyncCallbacks
+ .OpenReadOnlyManagedLedgerCallback() {
+ @Override
+ public void
openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl
readOnlyManagedLedger,
+ Object
ctx) {
+
finalPersistentSnapshotIndexes.getIndexList().forEach(index -> {
+ CompletableFuture<Void> handleSegmentFuture =
new CompletableFuture<>();
+ completableFutures.add(handleSegmentFuture);
+ readOnlyManagedLedger.asyncReadEntry(
+ new
PositionImpl(index.getSegmentLedgerID(),
+ index.getSegmentEntryID()),
+ new AsyncCallbacks.ReadEntryCallback()
{
+ @Override
+ public void
readEntryComplete(Entry entry, Object ctx) {
+
handleSnapshotSegmentEntry(entry);
+ indexes.put(new PositionImpl(
+
index.abortedMarkLedgerID,
+
index.abortedMarkEntryID),
+ index);
+ entry.release();
+
handleSegmentFuture.complete(null);
+ }
+
+ @Override
+ public void
readEntryFailed(ManagedLedgerException exception, Object ctx) {
+ if (exception instanceof
ManagedLedgerException
+
.NonRecoverableLedgerException) {
+ /*
+ The logic flow of
deleting expired segment is:
+ <p>
+ 1. delete segment
+ 2. update segment
index
+ </p>
+ If the worker delete
segment successfully
+ but failed to update
segment index,
+ the segment can not be
read according to the index.
+ We update index again if
there are invalid indexes.
+ */
+ if
(((ManagedLedgerImpl)topic.getManagedLedger())
+
.ledgerExists(index.getAbortedMarkLedgerID())) {
+ log.error("[{}] Failed
to read snapshot segment [{}:{}]",
+
topic.getName(), index.segmentLedgerID,
+
index.segmentEntryID, exception);
+
handleSegmentFuture.completeExceptionally(exception);
+ } else {
+
hasInvalidIndex.set(true);
+ }
+ }
+ }
+ }, null);
+ });
+ openManagedLedgerFuture.complete(null);
+ }
+
+ @Override
+ public void
openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) {
+ log.error("[{}] Failed to open readOnly managed
ledger", topic, exception);
+
openManagedLedgerFuture.completeExceptionally(exception);
+ }
+ };
+
+ TopicName snapshotIndexTopicName =
TopicName.get(TopicDomain.persistent.toString(),
+
TopicName.get(topic.getName()).getNamespaceObject(),
+
SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS);
+
this.topic.getBrokerService().getPulsar().getManagedLedgerFactory()
+
.asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName
+ .getPersistenceNamingEncoding(),
callback,
+ topic.getManagedLedger().getConfig(),
+ null);
+ /*
+ Wait the processor recover completely and then allow TB
+ to recover the messages after the
startReadCursorPosition.
+ */
+ return openManagedLedgerFuture
+ .thenCompose((ignore) ->
FutureUtil.waitForAll(completableFutures))
+ .thenCompose((i) -> {
+ /*
+ Update the snapshot segment index if there
exist invalid indexes.
+ */
+ if (hasInvalidIndex.get()) {
+
persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, ()
+ -> persistentWorker
+
.updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(),
+
indexes.values().stream().toList()));
+ }
+ /*
+ If there is no segment index, the
persistent worker will write segment begin from 0.
+ */
+ if (indexes.size() != 0) {
+
persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1);
+ }
+ /*
+ Append the aborted txn IDs in the index
metadata
+ can keep the order of the aborted txn in the
aborts.
+ So that we can trim the expired snapshot
segment in aborts
+ according to the latest transaction IDs in
the segmentIndex.
+ */
+
convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts())
+ .forEach(txnID -> aborts.put(txnID,
txnID));
+ return
CompletableFuture.completedFuture(finalStartReadCursorPosition);
+ }).exceptionally(ex -> {
+ log.error("[{}] Failed to recover snapshot
segment", this.topic.getName(), ex);
+ return null;
+ });
+
+ },
topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
+ .getExecutor(this));
+ }
+
+ @Override
+ public CompletableFuture<Void> clearAbortedTxnSnapshot() {
+ return
persistentWorker.appendTask(PersistentWorker.OperationType.Clear,
+ persistentWorker::clearSnapshotSegmentAndIndexes);
+ }
+
+ @Override
+ public long getLastSnapshotTimestamps() {
+ return this.lastSnapshotTimestamps;
+ }
+
+ @Override
+ public CompletableFuture<Void> closeAsync() {
+ return persistentWorker.closeAsync();
+ }
+
+ private void handleSnapshotSegmentEntry(Entry entry) {
+ //decode snapshot from entry
+ ByteBuf headersAndPayload = entry.getDataBuffer();
+ //skip metadata
+ Commands.parseMessageMetadata(headersAndPayload);
+ TransactionBufferSnapshotSegment snapshotSegment =
Schema.AVRO(TransactionBufferSnapshotSegment.class)
+ .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer());
+
+ TxnIDData lastTxn =
snapshotSegment.getAborts().get(snapshotSegment.getAborts().size() - 1);
+ segmentIndex.put(new
PositionImpl(snapshotSegment.getPersistentPositionLedgerId(),
+ snapshotSegment.getPersistentPositionEntryId()),
+ new TxnID(lastTxn.getMostSigBits(),
lastTxn.getLeastSigBits()));
+ convertTypeToTxnID(snapshotSegment.getAborts()).forEach(txnID ->
aborts.put(txnID, txnID));
+ }
+
+ private long getSystemClientOperationTimeoutMs() throws Exception {
+ PulsarClientImpl pulsarClient = (PulsarClientImpl)
topic.getBrokerService().getPulsar().getClient();
+ return pulsarClient.getConfiguration().getOperationTimeoutMs();
+ }
+
+ private <T> void closeReader(SystemTopicClient.Reader<T> reader) {
+ reader.closeAsync().exceptionally(e -> {
+ log.error("[{}]Transaction buffer snapshot reader close error!",
topic.getName(), e);
+ return null;
+ });
+ }
+
+ /**
+ * The PersistentWorker be responsible for executing the persistent tasks,
including:
+ * <p>
+ * 1. Write snapshot segment --- Encapsulate a sealed snapshot segment
and persistent it.
+ * 2. Delete snapshot segment --- Evict expired snapshot segments.
+ * 3. Update snapshot indexes --- Update snapshot indexes after
writing or deleting snapshot segment
+ * or update snapshot indexes metadata
regularly.
+ * 4. Clear all snapshot segments and indexes. --- Executed when
deleting this topic.
+ * </p>
+ * * Task 1 and task 2 will be put into a task queue. The tasks in the
queue will be executed in order.
+ * * If the task queue is empty, task 3 will be executed immediately when
it is appended to the worker.
+ * Else, the worker will try to execute the tasks in the task queue.
+ * * When task 4 was appended into worker, the worker will change the
operation state to closed
+ * and cancel all tasks in the task queue. finally, execute the task 4
(clear task).
+ * If there are race conditions, throw an Exception to let users try again.
+ */
+ public class PersistentWorker {
+ protected final AtomicLong sequenceID = new AtomicLong(0);
+
+ private final PersistentTopic topic;
+
+ //Persistent snapshot segment and index at the single thread.
+ private final
CompletableFuture<SystemTopicClient.Writer<TransactionBufferSnapshotSegment>>
+ snapshotSegmentsWriterFuture;
+ private final
CompletableFuture<SystemTopicClient.Writer<TransactionBufferSnapshotIndexes>>
+ snapshotIndexWriterFuture;
+
+ private enum OperationState {
+ None,
+ Operating,
+ Closed
+ }
+ private static final AtomicReferenceFieldUpdater<PersistentWorker,
PersistentWorker.OperationState>
+ STATE_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class,
+ PersistentWorker.OperationState.class, "operationState");
+
+ public enum OperationType {
+ UpdateIndex,
+ WriteSegment,
+ DeleteSegment,
+ Clear
+ }
+
+ private volatile OperationState operationState = OperationState.None;
+
+ ConcurrentLinkedDeque<Pair<OperationType, Pair<CompletableFuture<Void>,
+ Supplier<CompletableFuture<Void>>>>> taskQueue = new
ConcurrentLinkedDeque<>();
+
+ public PersistentWorker(PersistentTopic topic) {
+ this.topic = topic;
+ this.snapshotSegmentsWriterFuture =
this.topic.getBrokerService().getPulsar()
+ .getTransactionBufferSnapshotServiceFactory()
+
.getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName()))
+ .exceptionally(ex -> {
+ log.error("{} Failed to create snapshot index writer",
topic.getName());
+ topic.close();
+ return null;
+ });
+ this.snapshotIndexWriterFuture =
this.topic.getBrokerService().getPulsar()
+ .getTransactionBufferSnapshotServiceFactory()
+
.getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName()))
+ .exceptionally((ex) -> {
+ log.error("{} Failed to create snapshot writer",
topic.getName());
+ topic.close();
+ return null;
+ });
+ }
+
+ public CompletableFuture<Void> appendTask(OperationType operationType,
+
Supplier<CompletableFuture<Void>> task) {
+ CompletableFuture<Void> taskExecutedResult = new
CompletableFuture<>();
+ switch (operationType) {
+ case UpdateIndex -> {
+ /*
+ The update index operation can be canceled when the task
queue is not empty,
+ so it should be executed immediately instead of
appending to the task queue.
+ If the taskQueue is not empty, the worker will execute
the tasks in the queue.
+ */
+ if (!taskQueue.isEmpty()) {
+
topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
Review Comment:
I have fogoted, why we need executor to excute this task? please add a note
for it
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java:
##########
@@ -0,0 +1,721 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,2
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.buffer.impl;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.Supplier;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl;
+import org.apache.commons.collections4.map.LinkedMap;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.systopic.SystemTopicClient;
+import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata;
+import
org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment;
+import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.apache.pulsar.common.naming.SystemTopicNames;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.protocol.Commands;
+import org.apache.pulsar.common.util.FutureUtil;
+
+@Slf4j
+public class SnapshotSegmentAbortedTxnProcessorImpl implements
AbortedTxnProcessor {
+
+ /**
+ * Stored the unsealed aborted transaction IDs Whose size is always less
than the snapshotSegmentCapacity.
+ * It will be persistent as a snapshot segment when its size reach the
configured capacity.
+ */
+ private LinkedList<TxnID> unsealedTxnIds;
+
+ /**
+ * The map is used to clear the aborted transaction IDs persistent in the
expired ledger.
+ * <p>
+ * The key PositionImpl {@link PositionImpl} is the persistent
position of
+ * the latest transaction of a segment.
+ * The value TxnID {@link TxnID} is the latest Transaction ID in a
segment.
+ * </p>
+ *
+ * <p>
+ * If the position is expired, the processor can get the according
latest
+ * transaction ID in this map. And then the processor can clear all the
+ * transaction IDs in the aborts {@link
SnapshotSegmentAbortedTxnProcessorImpl#aborts}
+ * that lower than the transaction ID.
+ * And then the processor can delete the segments persistently
according to
+ * the positions.
+ * </p>
+ */
+ private final LinkedMap<PositionImpl, TxnID> segmentIndex = new
LinkedMap<>();
+
+ /**
+ * This map is used to check whether a transaction is an aborted
transaction.
+ * <p>
+ * The transaction IDs is appended in order, so the processor can
delete expired
+ * transaction IDs according to the latest expired transaction IDs in
segmentIndex
+ * {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}.
+ * </p>
+ */
+ private final LinkedMap<TxnID, TxnID> aborts = new LinkedMap<>();
+ /**
+ * This map stores the indexes of the snapshot segment.
+ * <p>
+ * The key is the persistent position of the marker of the last
transaction in the segment.
+ * The value TransactionBufferSnapshotIndex {@link
TransactionBufferSnapshotIndex} is the
+ * indexes of the snapshot segment.
+ * </p>
+ */
+ private final LinkedMap<PositionImpl, TransactionBufferSnapshotIndex>
indexes = new LinkedMap<>();
+
+ private final PersistentTopic topic;
+
+ private volatile long lastSnapshotTimestamps;
+
+ /**
+ * The number of the aborted transaction IDs in a segment.
+ * This is calculated according to the configured memory size.
+ */
+ private final int snapshotSegmentCapacity;
+ /**
+ * Responsible for executing the persistent tasks.
+ * <p>Including:</p>
+ * <p> Update segment index.</p>
+ * <p> Write snapshot segment.</p>
+ * <p> Delete snapshot segment.</p>
+ * <p> Clear all snapshot segment. </p>
+ */
+ private final PersistentWorker persistentWorker;
+
+ private static final String SNAPSHOT_PREFIX = "multiple-";
+
+ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) {
+ this.topic = topic;
+ this.persistentWorker = new PersistentWorker(topic);
+ /*
+ Calculate the segment capital according to its size configuration.
+ <p>
+ The empty transaction segment size is 5.
+ Adding an empty linkedList, the size increase to 6.
+ Add the topic name the size increase to the 7 +
topic.getName().length().
+ Add the aborted transaction IDs, the size increase to 8 +
+ topic.getName().length() + 3 * aborted transaction ID size.
+ </p>
+ */
+ this.snapshotSegmentCapacity = (topic.getBrokerService().getPulsar()
+ .getConfiguration().getTransactionBufferSnapshotSegmentSize()
- 8 - topic.getName().length()) / 3;
+ this.unsealedTxnIds = new LinkedList<>();
+ }
+
+ @Override
+ public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) {
+ unsealedTxnIds.add(txnID);
+ aborts.put(txnID, txnID);
+ /*
+ The size of lastAbortedTxns reaches the configuration of the size
of snapshot segment.
+ Append a task to persistent the segment with the aborted
transaction IDs and the latest
+ transaction mark persistent position passed by param.
+ */
+ if (unsealedTxnIds.size() >= snapshotSegmentCapacity) {
+ LinkedList<TxnID> abortedSegment = unsealedTxnIds;
+ segmentIndex.put(position, txnID);
+
persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () ->
+ persistentWorker.takeSnapshotSegmentAsync(abortedSegment,
position));
+ this.unsealedTxnIds = new LinkedList<>();
+ }
+ }
+
+ @Override
+ public boolean checkAbortedTransaction(TxnID txnID) {
+ return aborts.containsKey(txnID);
+ }
+
+ /**
+ * Check werther the position in segmentIndex {@link
SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}
+ * is expired. If the position is not exist in the original topic, the
according transaction is an invalid
+ * transaction. And the according segment is invalid, too. The transaction
IDs before the transaction ID
+ * in the aborts are invalid, too.
+ */
+ @Override
+ public void trimExpiredAbortedTxns() {
+ //Checking whether there are some segment expired.
+ List<PositionImpl> positionsNeedToDelete = new ArrayList<>();
+ while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl)
topic.getManagedLedger())
+ .ledgerExists(segmentIndex.firstKey().getLedgerId())) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Topic transaction buffer clear aborted
transactions, maxReadPosition : {}",
+ topic.getName(), segmentIndex.firstKey());
+ }
+ PositionImpl positionNeedToDelete = segmentIndex.firstKey();
+ positionsNeedToDelete.add(positionNeedToDelete);
+
+ TxnID theLatestDeletedTxnID = segmentIndex.remove(0);
+ while (!aborts.firstKey().equals(theLatestDeletedTxnID)) {
+ aborts.remove(0);
+ }
+ aborts.remove(0);
+ }
+ //Batch delete the expired segment
+ if (!positionsNeedToDelete.isEmpty()) {
+
persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment,
+ () ->
persistentWorker.deleteSnapshotSegment(positionsNeedToDelete));
+ }
+ }
+
+ private String buildKey(long sequenceId) {
+ return SNAPSHOT_PREFIX + sequenceId + "-" + this.topic.getName();
+ }
+
+ @Override
+ public CompletableFuture<Void> takeAbortedTxnsSnapshot(PositionImpl
maxReadPosition) {
+ //Store the latest aborted transaction IDs in unsealedTxnIDs and the
according the latest max read position.
+ TransactionBufferSnapshotIndexesMetadata metadata = new
TransactionBufferSnapshotIndexesMetadata(
+ maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(),
+ convertTypeToTxnIDData(unsealedTxnIds));
+ return
persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () ->
persistentWorker
+ .updateSnapshotIndex(metadata,
indexes.values().stream().toList()));
+ }
+
+ @Override
+ public CompletableFuture<PositionImpl> recoverFromSnapshot() {
+ return
topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory()
+ .getTxnBufferSnapshotIndexService()
+
.createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> {
+ PositionImpl startReadCursorPosition = null;
+ TransactionBufferSnapshotIndexes persistentSnapshotIndexes
= null;
+ boolean hasIndex = false;
+ try {
+ /*
+ Read the transaction snapshot segment index.
+ <p>
+ The processor can get the sequence ID, unsealed
transaction IDs,
+ segment index list and max read position in the
snapshot segment index.
+ Then we can traverse the index list to read all
aborted transaction IDs
+ in segments to aborts.
+ </p>
+ */
+ while (reader.hasMoreEvents()) {
+ Message<TransactionBufferSnapshotIndexes> message
= reader.readNextAsync()
+ .get(getSystemClientOperationTimeoutMs(),
TimeUnit.MILLISECONDS);
+ if (topic.getName().equals(message.getKey())) {
+ TransactionBufferSnapshotIndexes
transactionBufferSnapshotIndexes = message.getValue();
+ if (transactionBufferSnapshotIndexes != null) {
+ hasIndex = true;
+ persistentSnapshotIndexes =
transactionBufferSnapshotIndexes;
+ startReadCursorPosition = PositionImpl.get(
+
transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(),
+
transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId());
+ }
+ }
+ }
+ } catch (TimeoutException ex) {
+ Throwable t = FutureUtil.unwrapCompletionException(ex);
+ String errorMessage = String.format("[%s] Transaction
buffer recover fail by read "
+ + "transactionBufferSnapshot timeout!",
topic.getName());
+ log.error(errorMessage, t);
+ return FutureUtil.failedFuture(
+ new
BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t));
+ } catch (Exception ex) {
+ log.error("[{}] Transaction buffer recover fail when
read "
+ + "transactionBufferSnapshot!",
topic.getName(), ex);
+ return FutureUtil.failedFuture(ex);
+ } finally {
+ closeReader(reader);
+ }
+ PositionImpl finalStartReadCursorPosition =
startReadCursorPosition;
+ TransactionBufferSnapshotIndexes
finalPersistentSnapshotIndexes = persistentSnapshotIndexes;
+ if (!hasIndex) {
+ return CompletableFuture.completedFuture(null);
+ } else {
+ this.unsealedTxnIds =
convertTypeToTxnID(persistentSnapshotIndexes
+ .getSnapshot().getAborts());
+ }
+ //Read snapshot segment to recover aborts.
+ ArrayList<CompletableFuture<Void>> completableFutures =
new ArrayList<>();
+ CompletableFuture<Void> openManagedLedgerFuture = new
CompletableFuture<>();
+ AtomicBoolean hasInvalidIndex = new AtomicBoolean(false);
+ AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback
= new AsyncCallbacks
+ .OpenReadOnlyManagedLedgerCallback() {
+ @Override
+ public void
openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl
readOnlyManagedLedger,
+ Object
ctx) {
+
finalPersistentSnapshotIndexes.getIndexList().forEach(index -> {
+ CompletableFuture<Void> handleSegmentFuture =
new CompletableFuture<>();
+ completableFutures.add(handleSegmentFuture);
+ readOnlyManagedLedger.asyncReadEntry(
+ new
PositionImpl(index.getSegmentLedgerID(),
+ index.getSegmentEntryID()),
+ new AsyncCallbacks.ReadEntryCallback()
{
+ @Override
+ public void
readEntryComplete(Entry entry, Object ctx) {
+
handleSnapshotSegmentEntry(entry);
+ indexes.put(new PositionImpl(
+
index.abortedMarkLedgerID,
+
index.abortedMarkEntryID),
+ index);
+ entry.release();
+
handleSegmentFuture.complete(null);
+ }
+
+ @Override
+ public void
readEntryFailed(ManagedLedgerException exception, Object ctx) {
+ if (exception instanceof
ManagedLedgerException
+
.NonRecoverableLedgerException) {
+ /*
+ The logic flow of
deleting expired segment is:
+ <p>
+ 1. delete segment
+ 2. update segment
index
+ </p>
+ If the worker delete
segment successfully
+ but failed to update
segment index,
+ the segment can not be
read according to the index.
+ We update index again if
there are invalid indexes.
+ */
+ if
(((ManagedLedgerImpl)topic.getManagedLedger())
+
.ledgerExists(index.getAbortedMarkLedgerID())) {
+ log.error("[{}] Failed
to read snapshot segment [{}:{}]",
+
topic.getName(), index.segmentLedgerID,
+
index.segmentEntryID, exception);
+
handleSegmentFuture.completeExceptionally(exception);
+ } else {
+
hasInvalidIndex.set(true);
+ }
+ }
+ }
+ }, null);
+ });
+ openManagedLedgerFuture.complete(null);
+ }
+
+ @Override
+ public void
openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) {
+ log.error("[{}] Failed to open readOnly managed
ledger", topic, exception);
+
openManagedLedgerFuture.completeExceptionally(exception);
+ }
+ };
+
+ TopicName snapshotIndexTopicName =
TopicName.get(TopicDomain.persistent.toString(),
+
TopicName.get(topic.getName()).getNamespaceObject(),
+
SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS);
+
this.topic.getBrokerService().getPulsar().getManagedLedgerFactory()
+
.asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName
+ .getPersistenceNamingEncoding(),
callback,
+ topic.getManagedLedger().getConfig(),
+ null);
+ /*
+ Wait the processor recover completely and then allow TB
+ to recover the messages after the
startReadCursorPosition.
+ */
+ return openManagedLedgerFuture
+ .thenCompose((ignore) ->
FutureUtil.waitForAll(completableFutures))
+ .thenCompose((i) -> {
+ /*
+ Update the snapshot segment index if there
exist invalid indexes.
+ */
+ if (hasInvalidIndex.get()) {
+
persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, ()
+ -> persistentWorker
+
.updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(),
+
indexes.values().stream().toList()));
+ }
+ /*
+ If there is no segment index, the
persistent worker will write segment begin from 0.
+ */
+ if (indexes.size() != 0) {
+
persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1);
+ }
+ /*
+ Append the aborted txn IDs in the index
metadata
+ can keep the order of the aborted txn in the
aborts.
+ So that we can trim the expired snapshot
segment in aborts
+ according to the latest transaction IDs in
the segmentIndex.
+ */
+
convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts())
+ .forEach(txnID -> aborts.put(txnID,
txnID));
+ return
CompletableFuture.completedFuture(finalStartReadCursorPosition);
+ }).exceptionally(ex -> {
+ log.error("[{}] Failed to recover snapshot
segment", this.topic.getName(), ex);
+ return null;
+ });
+
+ },
topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
+ .getExecutor(this));
+ }
+
+ @Override
+ public CompletableFuture<Void> clearAbortedTxnSnapshot() {
+ return
persistentWorker.appendTask(PersistentWorker.OperationType.Clear,
+ persistentWorker::clearSnapshotSegmentAndIndexes);
+ }
+
+ @Override
+ public long getLastSnapshotTimestamps() {
+ return this.lastSnapshotTimestamps;
+ }
+
+ @Override
+ public CompletableFuture<Void> closeAsync() {
+ return persistentWorker.closeAsync();
+ }
+
+ private void handleSnapshotSegmentEntry(Entry entry) {
+ //decode snapshot from entry
+ ByteBuf headersAndPayload = entry.getDataBuffer();
+ //skip metadata
+ Commands.parseMessageMetadata(headersAndPayload);
+ TransactionBufferSnapshotSegment snapshotSegment =
Schema.AVRO(TransactionBufferSnapshotSegment.class)
+ .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer());
+
+ TxnIDData lastTxn =
snapshotSegment.getAborts().get(snapshotSegment.getAborts().size() - 1);
+ segmentIndex.put(new
PositionImpl(snapshotSegment.getPersistentPositionLedgerId(),
+ snapshotSegment.getPersistentPositionEntryId()),
+ new TxnID(lastTxn.getMostSigBits(),
lastTxn.getLeastSigBits()));
+ convertTypeToTxnID(snapshotSegment.getAborts()).forEach(txnID ->
aborts.put(txnID, txnID));
+ }
+
+ private long getSystemClientOperationTimeoutMs() throws Exception {
+ PulsarClientImpl pulsarClient = (PulsarClientImpl)
topic.getBrokerService().getPulsar().getClient();
+ return pulsarClient.getConfiguration().getOperationTimeoutMs();
+ }
+
+ private <T> void closeReader(SystemTopicClient.Reader<T> reader) {
+ reader.closeAsync().exceptionally(e -> {
+ log.error("[{}]Transaction buffer snapshot reader close error!",
topic.getName(), e);
+ return null;
+ });
+ }
+
+ /**
+ * The PersistentWorker be responsible for executing the persistent tasks,
including:
+ * <p>
+ * 1. Write snapshot segment --- Encapsulate a sealed snapshot segment
and persistent it.
+ * 2. Delete snapshot segment --- Evict expired snapshot segments.
+ * 3. Update snapshot indexes --- Update snapshot indexes after
writing or deleting snapshot segment
+ * or update snapshot indexes metadata
regularly.
+ * 4. Clear all snapshot segments and indexes. --- Executed when
deleting this topic.
+ * </p>
+ * * Task 1 and task 2 will be put into a task queue. The tasks in the
queue will be executed in order.
+ * * If the task queue is empty, task 3 will be executed immediately when
it is appended to the worker.
+ * Else, the worker will try to execute the tasks in the task queue.
+ * * When task 4 was appended into worker, the worker will change the
operation state to closed
+ * and cancel all tasks in the task queue. finally, execute the task 4
(clear task).
+ * If there are race conditions, throw an Exception to let users try again.
+ */
+ public class PersistentWorker {
+ protected final AtomicLong sequenceID = new AtomicLong(0);
+
+ private final PersistentTopic topic;
+
+ //Persistent snapshot segment and index at the single thread.
+ private final
CompletableFuture<SystemTopicClient.Writer<TransactionBufferSnapshotSegment>>
+ snapshotSegmentsWriterFuture;
+ private final
CompletableFuture<SystemTopicClient.Writer<TransactionBufferSnapshotIndexes>>
+ snapshotIndexWriterFuture;
+
+ private enum OperationState {
+ None,
+ Operating,
+ Closed
+ }
+ private static final AtomicReferenceFieldUpdater<PersistentWorker,
PersistentWorker.OperationState>
+ STATE_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class,
+ PersistentWorker.OperationState.class, "operationState");
+
+ public enum OperationType {
+ UpdateIndex,
+ WriteSegment,
+ DeleteSegment,
+ Clear
+ }
+
+ private volatile OperationState operationState = OperationState.None;
+
+ ConcurrentLinkedDeque<Pair<OperationType, Pair<CompletableFuture<Void>,
+ Supplier<CompletableFuture<Void>>>>> taskQueue = new
ConcurrentLinkedDeque<>();
+
+ public PersistentWorker(PersistentTopic topic) {
+ this.topic = topic;
+ this.snapshotSegmentsWriterFuture =
this.topic.getBrokerService().getPulsar()
+ .getTransactionBufferSnapshotServiceFactory()
+
.getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName()))
+ .exceptionally(ex -> {
+ log.error("{} Failed to create snapshot index writer",
topic.getName());
+ topic.close();
+ return null;
+ });
+ this.snapshotIndexWriterFuture =
this.topic.getBrokerService().getPulsar()
+ .getTransactionBufferSnapshotServiceFactory()
+
.getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName()))
+ .exceptionally((ex) -> {
+ log.error("{} Failed to create snapshot writer",
topic.getName());
+ topic.close();
+ return null;
+ });
+ }
+
+ public CompletableFuture<Void> appendTask(OperationType operationType,
+
Supplier<CompletableFuture<Void>> task) {
+ CompletableFuture<Void> taskExecutedResult = new
CompletableFuture<>();
+ switch (operationType) {
+ case UpdateIndex -> {
+ /*
+ The update index operation can be canceled when the task
queue is not empty,
+ so it should be executed immediately instead of
appending to the task queue.
+ If the taskQueue is not empty, the worker will execute
the tasks in the queue.
+ */
+ if (!taskQueue.isEmpty()) {
+
topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
+ .getExecutor(this).submit(this::executeTask);
+ return CompletableFuture.completedFuture(null);
+ } else if (STATE_UPDATER.compareAndSet(this,
OperationState.None, OperationState.Operating)) {
+ return task.get().whenComplete((ignore, throwable) -> {
+ if (throwable != null && log.isDebugEnabled()) {
+ log.debug("[{}] Failed to update index
snapshot", topic.getName(), throwable);
+ }
+ STATE_UPDATER.compareAndSet(this,
OperationState.Operating, OperationState.None);
+ });
+ } else {
+ return CompletableFuture.completedFuture(null);
+ }
+ }
+ /*
+ Only the operations of WriteSegment and DeleteSegment will
be appended into the taskQueue.
+ The operation will be canceled when the worker is close
which means the topic is deleted.
+ */
+ case WriteSegment, DeleteSegment -> {
+ if
(!STATE_UPDATER.get(this).equals(OperationState.Closed)) {
+ taskQueue.add(new MutablePair<>(operationType, new
MutablePair<>(taskExecutedResult, task)));
+ executeTask();
+ return taskExecutedResult;
+ } else {
+ return CompletableFuture.completedFuture(null);
+ }
+ }
+ case Clear -> {
+ /*
+ Do not clear the snapshots if the topic is used.
+ If the users want to delete a topic, they should stop
the usage of the topic.
+ */
+ if (STATE_UPDATER.compareAndSet(this, OperationState.None,
OperationState.Closed)) {
+ taskQueue.forEach(pair ->
+
pair.getRight().getRight().get().completeExceptionally(
+ new
BrokerServiceException.TransactionBufferClosedException(
+ String.format("Cancel the
operation [%s] due to the"
+ + "
transaction buffer of the topic[%s] already closed",
+ pair.getLeft().name(),
this.topic.getName()))));
+ taskQueue.clear();
+ /*
+ The task of clear all snapshot segments and indexes
is executed immediately.
+ */
+ return task.get();
+ } else {
+ return FutureUtil.failedFuture(
+ new BrokerServiceException.NotAllowedException(
+ String.format("Failed to clear the
snapshot of topic [%s] due to "
+ + "the topic is used. Please
stop the using of the topic "
+ + "and try it again",
this.topic.getName())));
+ }
+ }
+ default -> {
+ return FutureUtil.failedFuture(new BrokerServiceException
+ .NotAllowedException(String.format("Th operation
[%s] is unsupported",
+ operationType.name())));
+ }
+ }
+ }
+
+ private void executeTask() {
+ if (taskQueue.isEmpty()) return;
+ if (STATE_UPDATER.compareAndSet(this, OperationState.None,
OperationState.Operating)) {
+ Pair<OperationType, Pair<CompletableFuture<Void>,
Supplier<CompletableFuture<Void>>>> firstTask =
+ taskQueue.getFirst();
+ if (firstTask == null) return;
+ firstTask.getValue().getRight().get().whenComplete((ignore,
throwable) -> {
+ if (throwable != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Failed to do operation do
operation of [{}]",
+ topic.getName(),
firstTask.getKey().name(), throwable);
+ }
+
firstTask.getRight().getKey().completeExceptionally(throwable);
+ } else {
+ firstTask.getRight().getKey().complete(null);
+ taskQueue.removeFirst();
+ }
+ STATE_UPDATER.compareAndSet(this, OperationState.Operating,
+ OperationState.None);
+ //Execute the next task in the task queue.
+
topic.getBrokerService().getPulsar().getTransactionExecutorProvider()
+ .getExecutor(this).submit(this::executeTask);
+ });
+ }
+ }
+
+ private CompletableFuture<Void>
takeSnapshotSegmentAsync(LinkedList<TxnID> sealedAbortedTxnIdSegment,
+ PositionImpl
abortedMarkerPersistentPosition) {
+ return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment,
abortedMarkerPersistentPosition).thenRun(() -> {
+ if (log.isDebugEnabled()) {
+ log.debug("Successes to take snapshot segment [{}] at
maxReadPosition [{}] "
+ + "for the topic [{}], and the size of the
segment is [{}]",
+ this.sequenceID, abortedMarkerPersistentPosition,
topic.getName(), sealedAbortedTxnIdSegment.size());
+ }
+ this.sequenceID.getAndIncrement();
+ }).exceptionally(e -> {
+ //Just log the error, and the processor will try to take
snapshot again when the transactionBuffer
+ //append aborted txn nex time.
+ log.error("Failed to take snapshot segment [{}] at
maxReadPosition [{}] "
+ + "for the topic [{}], and the size of the
segment is [{}]",
+ this.sequenceID, abortedMarkerPersistentPosition,
topic.getName(), sealedAbortedTxnIdSegment.size(), e);
+ return null;
+ });
+ }
+
+ private CompletableFuture<Void>
writeSnapshotSegmentAsync(LinkedList<TxnID> segment,
+ PositionImpl
abortedMarkerPersistentPosition) {
+ TransactionBufferSnapshotSegment transactionBufferSnapshotSegment
= new TransactionBufferSnapshotSegment();
+
transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment));
+
transactionBufferSnapshotSegment.setTopicName(this.topic.getName());
+
transactionBufferSnapshotSegment.setPersistentPositionEntryId(abortedMarkerPersistentPosition.getEntryId());
+
transactionBufferSnapshotSegment.setPersistentPositionLedgerId(abortedMarkerPersistentPosition.getLedgerId());
+
+ return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> {
+
transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get());
+ return
segmentWriter.writeAsync(buildKey(this.sequenceID.get()),
transactionBufferSnapshotSegment);
+ }).thenCompose((messageId) -> {
+ PositionImpl maxReadPosition = topic.getMaxReadPosition();
+ //Build index for this segment
+ TransactionBufferSnapshotIndex index = new
TransactionBufferSnapshotIndex();
+
index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId());
+
index.setAbortedMarkLedgerID(abortedMarkerPersistentPosition.getLedgerId());
+
index.setAbortedMarkEntryID(abortedMarkerPersistentPosition.getEntryId());
+ index.setSegmentLedgerID(((MessageIdImpl)
messageId).getLedgerId());
+ index.setSegmentEntryID(((MessageIdImpl)
messageId).getEntryId());
+
+ indexes.put(abortedMarkerPersistentPosition, index);
+ //update snapshot segment index.
+ return updateSnapshotIndex(new
TransactionBufferSnapshotIndexesMetadata(
+ maxReadPosition.getLedgerId(),
maxReadPosition.getEntryId(), new LinkedList<>()),
+ indexes.values().stream().toList());
+ });
+ }
+
+ // update index after delete all segment.
+ private CompletableFuture<Void>
deleteSnapshotSegment(List<PositionImpl> positionNeedToDeletes) {
+ List<CompletableFuture<Void>> results = new ArrayList<>();
+ for (PositionImpl positionNeedToDelete : positionNeedToDeletes) {
+ long sequenceIdNeedToDelete =
indexes.get(positionNeedToDelete).getSequenceID();
+ results.add(snapshotSegmentsWriterFuture
+ .thenCompose(writer ->
writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null))
Review Comment:
If one segment has been written to the segment topic but the segment has not
been written into the index and updated success, how can clear this segment?
##########
pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java:
##########
@@ -0,0 +1,147 @@
+package org.apache.pulsar.broker.transaction;
+
+import java.lang.reflect.Field;
+import java.util.LinkedList;
+import java.util.NavigableMap;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.commons.collections4.map.LinkedMap;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
+import
org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class AbortTxnProcessorTest extends TransactionTestBase {
Review Comment:
```suggestion
public class SegementAbortedTxnProcessorTest extends TransactionTestBase {
```
##########
pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java:
##########
@@ -818,9 +820,98 @@ public void
openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob
//verify snapshot
assertEquals(snapshot.getTopicName(), snapshotTopic);
assertEquals(snapshot.getSequenceId(), 2L);
- assertEquals(snapshot.getMaxReadPositionLedgerId(), 2L);
- assertEquals(snapshot.getMaxReadPositionEntryId(), 3L);
+ assertEquals(snapshot.getPersistentPositionLedgerId(), 2L);
+ assertEquals(snapshot.getPersistentPositionEntryId(), 3L);
assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1));
}
+ //Verify the snapshotSegmentProcessor end to end
+ @Test
+ public void testSnapshotSegment() throws Exception {
+ String topic = NAMESPACE1 + "/testSnapshotSegment";
+ String subName = "testSnapshotSegment";
+
+ LinkedMap<Transaction, MessageId> ongoingTxns = new LinkedMap<>();
+ LinkedList<MessageId> abortedTxns = new LinkedList<>();
+ // 0. Modify the configurations, enabling the segment snapshot and set
the size of the snapshot segment.
+ int theSizeOfSegment = 10;
+ int theCountOfSnapshotMaxTxnCount = 3;
+
this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true);
+
this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(theSizeOfSegment);
+ this.getPulsarServiceList().get(0).getConfig()
+
.setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount);
+ // 1. Build producer and consumer
+ Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
+ .topic(topic)
+ .enableBatching(false)
+ .create();
+
+ Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
+ .topic(topic)
+ .subscriptionName(subName)
+ .subscriptionType(SubscriptionType.Exclusive)
+ .subscribe();
+
+ // 2. Check the AbortedTxnProcessor workflow 10 times
+ int messageSize = theSizeOfSegment * 4;
+ for (int i = 0; i < 10; i++) {
+ MessageId maxReadMessage = null;
+ int abortedTxnSize = 0;
+ for (int j = 0; j < messageSize; j++) {
+ Transaction transaction = pulsarClient.newTransaction()
+ .withTransactionTimeout(5,
TimeUnit.MINUTES).build().get();
+ //Half common message and half transaction message.
+ if (j % 2 == 0) {
+ MessageId messageId =
producer.newMessage(transaction).value(i * 10 + j).send();
+ //And the transaction message have a half which are
aborted.
+ if (RandomUtils.nextInt() % 2 == 0) {
+ transaction.abort().get();
+ abortedTxns.add(messageId);
+ abortedTxnSize++;
+ } else {
+ ongoingTxns.put(transaction, messageId);
+ if (maxReadMessage == null) {
+ //The except number of the messages that can be
read
+ maxReadMessage = messageId;
+ }
+ }
+ } else {
+ producer.newMessage().value(i * 10 + j).send();
+ transaction.commit().get();
+ }
+ }
+ // 2.1 Receive all message before the maxReadPosition to verify
the correctness of the max read position.
+ int hasReceived = 0;
+ while (true) {
+ Message<Integer> message = consumer.receive(2,
TimeUnit.SECONDS);
+ if (message != null) {
+
Assert.assertTrue(message.getMessageId().compareTo(maxReadMessage) < 0);
+ hasReceived ++;
+ } else {
+ break;
+ }
+ }
+ //2.2 Commit all ongoing transaction and verify that the consumer
can receive all rest message
+ // expect for aborted txn message.
+ for (Transaction ongoingTxn: ongoingTxns.keySet()) {
+ ongoingTxn.commit().get();
+ }
+ ongoingTxns.clear();
+ for (int k = hasReceived; k < messageSize - abortedTxnSize; k++) {
+ Message<Integer> message = consumer.receive(2,
TimeUnit.SECONDS);
+ assertNotNull(message);
+ assertFalse(abortedTxns.contains(message.getMessageId()));
+ }
+ }
+ // 3. After the topic unload, the consumer can receive all the
messages in the 10 tests
+ // expect for the aborted transaction messages.
Review Comment:
before unload, check the aborted txn, segment, index stats from the
processor, after unload check again. and add the tirm and clear test.
##########
pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java:
##########
@@ -818,9 +820,98 @@ public void
openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob
//verify snapshot
assertEquals(snapshot.getTopicName(), snapshotTopic);
assertEquals(snapshot.getSequenceId(), 2L);
- assertEquals(snapshot.getMaxReadPositionLedgerId(), 2L);
- assertEquals(snapshot.getMaxReadPositionEntryId(), 3L);
+ assertEquals(snapshot.getPersistentPositionLedgerId(), 2L);
+ assertEquals(snapshot.getPersistentPositionEntryId(), 3L);
assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1));
}
+ //Verify the snapshotSegmentProcessor end to end
+ @Test
+ public void testSnapshotSegment() throws Exception {
+ String topic = NAMESPACE1 + "/testSnapshotSegment";
+ String subName = "testSnapshotSegment";
+
+ LinkedMap<Transaction, MessageId> ongoingTxns = new LinkedMap<>();
+ LinkedList<MessageId> abortedTxns = new LinkedList<>();
+ // 0. Modify the configurations, enabling the segment snapshot and set
the size of the snapshot segment.
+ int theSizeOfSegment = 10;
Review Comment:
if the size is 10, how many aborted txn in one segment?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
