jeqo commented on code in PR #14127:
URL: https://github.com/apache/kafka/pull/14127#discussion_r1294016456
##########
storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java:
##########
@@ -64,302 +64,395 @@
class ConsumerTask implements Runnable, Closeable {
private static final Logger log =
LoggerFactory.getLogger(ConsumerTask.class);
- private static final long POLL_INTERVAL_MS = 100L;
+ static long pollIntervalMs = 100L;
private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
- private final KafkaConsumer<byte[], byte[]> consumer;
- private final String metadataTopicName;
+ private final Consumer<byte[], byte[]> consumer;
private final RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler;
private final RemoteLogMetadataTopicPartitioner topicPartitioner;
private final Time time;
- // It indicates whether the closing process has been started or not. If it
is set as true,
- // consumer will stop consuming messages, and it will not allow partition
assignments to be updated.
- private volatile boolean closing = false;
-
- // It indicates whether the consumer needs to assign the partitions or
not. This is set when it is
- // determined that the consumer needs to be assigned with the updated
partitions.
- private volatile boolean assignPartitions = false;
+ // It indicates whether the ConsumerTask is closed or not.
+ private volatile boolean isClosed = false;
+ // It indicates whether the user topic partition assignment to the
consumer has changed or not. If the assignment
+ // has changed, the consumer will eventually start tracking the newly
assigned partitions and stop tracking the
+ // ones it is no longer assigned to.
+ private volatile boolean hasAssignmentChanged = true;
// It represents a lock for any operations related to the
assignedTopicPartitions.
private final Object assignPartitionsLock = new Object();
// Remote log metadata topic partitions that consumer is assigned to.
- private volatile Set<Integer> assignedMetaPartitions =
Collections.emptySet();
+ private volatile Set<Integer> assignedMetadataPartitions =
Collections.emptySet();
// User topic partitions that this broker is a leader/follower for.
- private Set<TopicIdPartition> assignedTopicPartitions =
Collections.emptySet();
+ private volatile Map<TopicIdPartition, UserTopicIdPartition>
assignedUserTopicIdPartitions = Collections.emptyMap();
+ private volatile Set<TopicIdPartition>
processedAssignmentOfUserTopicIdPartitions = Collections.emptySet();
- // Map of remote log metadata topic partition to consumed offsets.
Received consumer records
- // may or may not have been processed based on the assigned topic
partitions.
- private final Map<Integer, Long> partitionToConsumedOffsets = new
ConcurrentHashMap<>();
+ private long uninitializedAt;
+ private boolean isAllUserTopicPartitionsInitialized;
- // Map of remote log metadata topic partition to processed offsets that
were synced in committedOffsetsFile.
- private Map<Integer, Long> lastSyncedPartitionToConsumedOffsets =
Collections.emptyMap();
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> readOffsetsByMetadataPartition = new
ConcurrentHashMap<>();
+ private final Map<TopicIdPartition, Long> readOffsetsByUserTopicPartition
= new HashMap<>();
Review Comment:
nit: reuse `latest` concept
```suggestion
private final Map<Integer, Long> latestOffsetsByMetadataPartition = new
ConcurrentHashMap<>();
private final Map<TopicIdPartition, Long>
latestOffsetsByUserTopicPartition = new HashMap<>();
```
##########
storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java:
##########
@@ -64,302 +64,395 @@
class ConsumerTask implements Runnable, Closeable {
private static final Logger log =
LoggerFactory.getLogger(ConsumerTask.class);
- private static final long POLL_INTERVAL_MS = 100L;
+ static long pollIntervalMs = 100L;
private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
- private final KafkaConsumer<byte[], byte[]> consumer;
- private final String metadataTopicName;
+ private final Consumer<byte[], byte[]> consumer;
private final RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler;
private final RemoteLogMetadataTopicPartitioner topicPartitioner;
private final Time time;
- // It indicates whether the closing process has been started or not. If it
is set as true,
- // consumer will stop consuming messages, and it will not allow partition
assignments to be updated.
- private volatile boolean closing = false;
-
- // It indicates whether the consumer needs to assign the partitions or
not. This is set when it is
- // determined that the consumer needs to be assigned with the updated
partitions.
- private volatile boolean assignPartitions = false;
+ // It indicates whether the ConsumerTask is closed or not.
+ private volatile boolean isClosed = false;
+ // It indicates whether the user topic partition assignment to the
consumer has changed or not. If the assignment
+ // has changed, the consumer will eventually start tracking the newly
assigned partitions and stop tracking the
+ // ones it is no longer assigned to.
+ private volatile boolean hasAssignmentChanged = true;
// It represents a lock for any operations related to the
assignedTopicPartitions.
private final Object assignPartitionsLock = new Object();
// Remote log metadata topic partitions that consumer is assigned to.
- private volatile Set<Integer> assignedMetaPartitions =
Collections.emptySet();
+ private volatile Set<Integer> assignedMetadataPartitions =
Collections.emptySet();
// User topic partitions that this broker is a leader/follower for.
- private Set<TopicIdPartition> assignedTopicPartitions =
Collections.emptySet();
+ private volatile Map<TopicIdPartition, UserTopicIdPartition>
assignedUserTopicIdPartitions = Collections.emptyMap();
+ private volatile Set<TopicIdPartition>
processedAssignmentOfUserTopicIdPartitions = Collections.emptySet();
- // Map of remote log metadata topic partition to consumed offsets.
Received consumer records
- // may or may not have been processed based on the assigned topic
partitions.
- private final Map<Integer, Long> partitionToConsumedOffsets = new
ConcurrentHashMap<>();
+ private long uninitializedAt;
+ private boolean isAllUserTopicPartitionsInitialized;
- // Map of remote log metadata topic partition to processed offsets that
were synced in committedOffsetsFile.
- private Map<Integer, Long> lastSyncedPartitionToConsumedOffsets =
Collections.emptyMap();
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> readOffsetsByMetadataPartition = new
ConcurrentHashMap<>();
+ private final Map<TopicIdPartition, Long> readOffsetsByUserTopicPartition
= new HashMap<>();
- private final long committedOffsetSyncIntervalMs;
- private CommittedOffsetsFile committedOffsetsFile;
- private long lastSyncedTimeMs;
+ private Map<TopicPartition, StartAndEndOffsetHolder>
offsetHolderByMetadataPartition = new HashMap<>();
+ private boolean isOffsetsFetchFailed = false;
+ private long lastFailedFetchOffsetsTimestamp;
- public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
- String metadataTopicName,
- RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
+ public ConsumerTask(RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
RemoteLogMetadataTopicPartitioner topicPartitioner,
- Path committedOffsetsPath,
- Time time,
- long committedOffsetSyncIntervalMs) {
- this.consumer = Objects.requireNonNull(consumer);
- this.metadataTopicName = Objects.requireNonNull(metadataTopicName);
+ Consumer<byte[], byte[]> consumer) {
+ this.consumer = consumer;
this.remotePartitionMetadataEventHandler =
Objects.requireNonNull(remotePartitionMetadataEventHandler);
this.topicPartitioner = Objects.requireNonNull(topicPartitioner);
- this.time = Objects.requireNonNull(time);
- this.committedOffsetSyncIntervalMs = committedOffsetSyncIntervalMs;
-
- initializeConsumerAssignment(committedOffsetsPath);
- }
-
- private void initializeConsumerAssignment(Path committedOffsetsPath) {
- try {
- committedOffsetsFile = new
CommittedOffsetsFile(committedOffsetsPath.toFile());
- } catch (IOException e) {
- throw new KafkaException(e);
- }
-
- Map<Integer, Long> committedOffsets = Collections.emptyMap();
- try {
- // Load committed offset and assign them in the consumer.
- committedOffsets = committedOffsetsFile.readEntries();
- } catch (IOException e) {
- // Ignore the error and consumer consumes from the earliest offset.
- log.error("Encountered error while building committed offsets from
the file. " +
- "Consumer will consume from the earliest offset
for the assigned partitions.", e);
- }
-
- if (!committedOffsets.isEmpty()) {
- // Assign topic partitions from the earlier committed offsets file.
- Set<Integer> earlierAssignedPartitions = committedOffsets.keySet();
- assignedMetaPartitions =
Collections.unmodifiableSet(earlierAssignedPartitions);
- Set<TopicPartition> metadataTopicPartitions =
earlierAssignedPartitions.stream()
-
.map(x -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, x))
-
.collect(Collectors.toSet());
- consumer.assign(metadataTopicPartitions);
-
- // Seek to the committed offsets
- for (Map.Entry<Integer, Long> entry : committedOffsets.entrySet())
{
- log.debug("Updating consumed offset: [{}] for partition [{}]",
entry.getValue(), entry.getKey());
- partitionToConsumedOffsets.put(entry.getKey(),
entry.getValue());
- consumer.seek(new
TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, entry.getKey()),
entry.getValue());
- }
-
- lastSyncedPartitionToConsumedOffsets =
Collections.unmodifiableMap(committedOffsets);
- }
+ this.time = new SystemTime();
+ this.uninitializedAt = time.milliseconds();
}
@Override
public void run() {
- log.info("Started Consumer task thread.");
- lastSyncedTimeMs = time.milliseconds();
- try {
- while (!closing) {
- maybeWaitForPartitionsAssignment();
+ log.info("Starting consumer task thread.");
+ while (!isClosed) {
+ try {
+ if (hasAssignmentChanged) {
+ maybeWaitForPartitionsAssignment();
+ }
log.trace("Polling consumer to receive remote log metadata
topic records");
- ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(POLL_INTERVAL_MS));
+ final ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(pollIntervalMs));
for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
processConsumerRecord(record);
}
-
- maybeSyncCommittedDataAndOffsets(false);
+ maybeMarkUserPartitionsAsReady();
+ } catch (final WakeupException ex) {
+ // ignore logging the error
+ isClosed = true;
+ } catch (final RetriableException ex) {
+ log.warn("Retriable error occurred while processing the
records. Retrying...", ex);
+ } catch (final Exception ex) {
+ isClosed = true;
+ log.error("Error occurred while processing the records", ex);
}
- } catch (Exception e) {
- log.error("Error occurred in consumer task, close:[{}]", closing,
e);
- } finally {
- maybeSyncCommittedDataAndOffsets(true);
- closeConsumer();
- log.info("Exiting from consumer task thread");
}
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (final Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ log.info("Exited from consumer task thread");
}
private void processConsumerRecord(ConsumerRecord<byte[], byte[]> record) {
- // Taking assignPartitionsLock here as updateAssignmentsForPartitions
changes assignedTopicPartitions
- // and also calls
remotePartitionMetadataEventHandler.clearTopicPartition(removedPartition) for
the removed
- // partitions.
- RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
- synchronized (assignPartitionsLock) {
- if
(assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
-
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
- } else {
- log.debug("This event {} is skipped as the topic partition is
not assigned for this instance.", remoteLogMetadata);
- }
- log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
- partitionToConsumedOffsets.put(record.partition(),
record.offset());
+ final RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
+ if (canProcess(remoteLogMetadata, record.offset())) {
+
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+
readOffsetsByUserTopicPartition.put(remoteLogMetadata.topicIdPartition(),
record.offset());
+ } else {
+ log.debug("The event {} is skipped because it is either already
processed or not assigned to this consumer", remoteLogMetadata);
}
+ log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
+ readOffsetsByMetadataPartition.put(record.partition(),
record.offset());
+ }
+
+ private boolean canProcess(final RemoteLogMetadata metadata, final long
recordOffset) {
+ final TopicIdPartition tpId = metadata.topicIdPartition();
+ final Long readOffset = readOffsetsByUserTopicPartition.get(tpId);
+ return processedAssignmentOfUserTopicIdPartitions.contains(tpId) &&
(readOffset == null || readOffset < recordOffset);
}
- private void maybeSyncCommittedDataAndOffsets(boolean forceSync) {
- // Return immediately if there is no consumption from last time.
- boolean noConsumedOffsetUpdates =
partitionToConsumedOffsets.equals(lastSyncedPartitionToConsumedOffsets);
- if (noConsumedOffsetUpdates || !forceSync && time.milliseconds() -
lastSyncedTimeMs < committedOffsetSyncIntervalMs) {
- log.debug("Skip syncing committed offsets,
noConsumedOffsetUpdates: {}, forceSync: {}", noConsumedOffsetUpdates,
forceSync);
+ private void maybeMarkUserPartitionsAsReady() {
+ if (isAllUserTopicPartitionsInitialized) {
return;
}
-
- try {
- // Need to take lock on assignPartitionsLock as
assignedTopicPartitions might
- // get updated by other threads.
- synchronized (assignPartitionsLock) {
- for (TopicIdPartition topicIdPartition :
assignedTopicPartitions) {
- int metadataPartition =
topicPartitioner.metadataPartition(topicIdPartition);
- Long offset =
partitionToConsumedOffsets.get(metadataPartition);
- if (offset != null) {
-
remotePartitionMetadataEventHandler.syncLogMetadataSnapshot(topicIdPartition,
metadataPartition, offset);
+ maybeFetchStartAndEndOffsets();
+ boolean isAllInitialized = true;
+ for (final UserTopicIdPartition utp :
assignedUserTopicIdPartitions.values()) {
+ if (utp.isAssigned && !utp.isInitialized) {
+ final Integer metadataPartition = utp.metadataPartition;
+ final StartAndEndOffsetHolder holder =
offsetHolderByMetadataPartition.get(toRemoteLogPartition(metadataPartition));
+ // The offset-holder can be null, when the recent assignment
wasn't picked up by the consumer.
+ if (holder != null) {
+ final Long readOffset =
readOffsetsByMetadataPartition.getOrDefault(metadataPartition, -1L);
+ // 1) The end-offset was fetched only once during
reassignment. The metadata-partition can receive
+ // new stream of records, so the consumer can read records
more than the last-fetched end-offset.
+ // 2) When the internal topic becomes empty due to breach
by size/time/start-offset, then there
+ // are no records to read.
+ if (readOffset + 1 >= holder.endOffset ||
holder.endOffset.equals(holder.startOffset)) {
+ markInitialized(utp);
} else {
- log.debug("Skipping sync-up of the
remote-log-metadata-file for partition: [{}] , with remote log metadata
partition{}, and no offset",
- topicIdPartition, metadataPartition);
+ log.debug("The user-topic-partition {} could not be
marked initialized since the read-offset is {} " +
+ "but the end-offset is {} for the
metadata-partition {}", utp, readOffset, holder.endOffset,
+ metadataPartition);
}
+ } else {
+ log.debug("The offset-holder is null for the
metadata-partition {}. The consumer may not have picked" +
+ " up the recent assignment", metadataPartition);
}
-
- // Write partitionToConsumedOffsets into committed offsets
file as we do not want to process them again
- // in case of restarts.
- committedOffsetsFile.writeEntries(partitionToConsumedOffsets);
- lastSyncedPartitionToConsumedOffsets = new
HashMap<>(partitionToConsumedOffsets);
}
-
- lastSyncedTimeMs = time.milliseconds();
- } catch (IOException e) {
- throw new KafkaException("Error encountered while writing
committed offsets to a local file", e);
+ isAllInitialized = isAllInitialized && utp.isInitialized;
}
- }
-
- private void closeConsumer() {
- log.info("Closing the consumer instance");
- try {
- consumer.close(Duration.ofSeconds(30));
- } catch (Exception e) {
- log.error("Error encountered while closing the consumer", e);
+ if (isAllInitialized) {
+ log.info("Initialized for all the {} assigned user-partitions
mapped to the {} meta-partitions in {} ms",
+ assignedUserTopicIdPartitions.size(),
assignedMetadataPartitions.size(),
+ time.milliseconds() - uninitializedAt);
}
+ isAllUserTopicPartitionsInitialized = isAllInitialized;
}
- private void maybeWaitForPartitionsAssignment() {
- Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ void maybeWaitForPartitionsAssignment() throws InterruptedException {
+ // Snapshots of the metadata-partition and user-topic-partition are
used to reduce the scope of the
+ // synchronization block.
+ // 1) LEADER_AND_ISR and STOP_REPLICA requests adds / removes the
user-topic-partitions from the request
+ // handler threads. Those threads should not be blocked for a long
time, therefore scope of the
+ // synchronization block is reduced to bare minimum.
+ // 2) Note that the consumer#position, consumer#seekToBeginning,
consumer#seekToEnd and the other consumer APIs
+ // response times are un-predictable. Those should not be kept in
the synchronization block.
+ final Set<Integer> metadataPartitionSnapshot = new HashSet<>();
+ final Set<UserTopicIdPartition> assignedUserTopicIdPartitionsSnapshot
= new HashSet<>();
synchronized (assignPartitionsLock) {
- // If it is closing, return immediately. This should be inside the
assignPartitionsLock as the closing is updated
- // in close() method with in the same lock to avoid any race
conditions.
- if (closing) {
- return;
+ while (!isClosed && assignedUserTopicIdPartitions.isEmpty()) {
+ log.debug("Waiting for remote log metadata partitions to be
assigned");
+ assignPartitionsLock.wait();
}
-
- while (assignedMetaPartitions.isEmpty()) {
- // If no partitions are assigned, wait until they are assigned.
- log.debug("Waiting for assigned remote log metadata
partitions..");
- try {
- // No timeout is set here, as it is always notified. Even
when it is closed, the race can happen
- // between the thread calling this method and the thread
calling close(). We should have a check
- // for closing as that might have been set and notified
with assignPartitionsLock by `close`
- // method.
- assignPartitionsLock.wait();
-
- if (closing) {
- return;
- }
- } catch (InterruptedException e) {
- throw new KafkaException(e);
- }
- }
-
- if (assignPartitions) {
- assignedMetaPartitionsSnapshot = new
HashSet<>(assignedMetaPartitions);
- // Removing unassigned meta partitions from
partitionToConsumedOffsets and partitionToCommittedOffsets
- partitionToConsumedOffsets.entrySet().removeIf(entry ->
!assignedMetaPartitions.contains(entry.getKey()));
-
- assignPartitions = false;
+ if (!isClosed && hasAssignmentChanged) {
+ assignedUserTopicIdPartitions.values().forEach(utp -> {
+ metadataPartitionSnapshot.add(utp.metadataPartition);
+ assignedUserTopicIdPartitionsSnapshot.add(utp);
+ });
+ hasAssignmentChanged = false;
}
}
-
- if (!assignedMetaPartitionsSnapshot.isEmpty()) {
- executeReassignment(assignedMetaPartitionsSnapshot);
+ if (!metadataPartitionSnapshot.isEmpty()) {
+ final Set<TopicPartition> remoteLogPartitions =
toRemoteLogPartitions(metadataPartitionSnapshot);
+ consumer.assign(remoteLogPartitions);
+ this.assignedMetadataPartitions =
Collections.unmodifiableSet(metadataPartitionSnapshot);
+ // for newly assigned user-partitions, read from the beginning of
the corresponding metadata partition
+ final Set<TopicPartition> seekToBeginOffsetPartitions =
assignedUserTopicIdPartitionsSnapshot
+ .stream()
+ .filter(utp -> !utp.isAssigned)
+ .map(utp -> toRemoteLogPartition(utp.metadataPartition))
+ .collect(Collectors.toSet());
+ consumer.seekToBeginning(seekToBeginOffsetPartitions);
+ // for other metadata partitions, read from the offset where the
processing left last time.
+ remoteLogPartitions.stream()
+ .filter(tp -> !seekToBeginOffsetPartitions.contains(tp) &&
+ readOffsetsByMetadataPartition.containsKey(tp.partition()))
+ .forEach(tp -> consumer.seek(tp,
readOffsetsByMetadataPartition.get(tp.partition())));
+ // mark all the user-topic-partitions as assigned to the consumer.
+ assignedUserTopicIdPartitionsSnapshot.forEach(utp -> {
+ if (!utp.isAssigned) {
+ // Note that there can be a race between `remove` and
`add` partition assignment. Calling the
+ // `maybeLoadPartition` here again to be sure that the
partition gets loaded on the handler.
+
remotePartitionMetadataEventHandler.maybeLoadPartition(utp.topicIdPartition);
+ utp.isAssigned = true;
+ }
+ });
+ processedAssignmentOfUserTopicIdPartitions =
assignedUserTopicIdPartitionsSnapshot.stream()
+ .map(utp -> utp.topicIdPartition).collect(Collectors.toSet());
+
clearResourcesForUnassignedUserTopicPartitions(assignedUserTopicIdPartitionsSnapshot);
+ isAllUserTopicPartitionsInitialized = false;
+ uninitializedAt = time.milliseconds();
+ fetchStartAndEndOffsets();
}
}
- private void executeReassignment(Set<Integer>
assignedMetaPartitionsSnapshot) {
- Set<TopicPartition> assignedMetaTopicPartitions =
- assignedMetaPartitionsSnapshot.stream()
- .map(partitionNum -> new
TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
- .collect(Collectors.toSet());
- log.info("Reassigning partitions to consumer task [{}]",
assignedMetaTopicPartitions);
- consumer.assign(assignedMetaTopicPartitions);
+ private void
clearResourcesForUnassignedUserTopicPartitions(Set<UserTopicIdPartition>
assignedUTPs) {
+ Set<TopicIdPartition> assignedPartitions = assignedUTPs.stream()
+ .map(utp -> utp.topicIdPartition).collect(Collectors.toSet());
+ // Note that there can be previously assigned user-topic-partitions
where no records are there to read
+ // (eg) none of the segments for a partition were uploaded. Those
partition resources won't be cleared.
+ // It can be fixed later when required since they are empty resources.
+ Set<TopicIdPartition> unassignedPartitions =
readOffsetsByUserTopicPartition.keySet()
+ .stream()
+ .filter(e -> !assignedPartitions.contains(e))
+ .collect(Collectors.toSet());
+ unassignedPartitions.forEach(unassignedPartition -> {
+
remotePartitionMetadataEventHandler.clearTopicPartition(unassignedPartition);
+ readOffsetsByUserTopicPartition.remove(unassignedPartition);
+ });
+ log.info("Unassigned user-topic-partitions: {}",
unassignedPartitions.size());
}
- public void addAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
- updateAssignmentsForPartitions(partitions, Collections.emptySet());
+ public void addAssignmentsForPartitions(final Set<TopicIdPartition>
partitions) {
+ updateAssignments(Objects.requireNonNull(partitions),
Collections.emptySet());
}
- public void removeAssignmentsForPartitions(Set<TopicIdPartition>
partitions) {
- updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ public void removeAssignmentsForPartitions(final Set<TopicIdPartition>
partitions) {
+ updateAssignments(Collections.emptySet(),
Objects.requireNonNull(partitions));
}
- private void updateAssignmentsForPartitions(Set<TopicIdPartition>
addedPartitions,
- Set<TopicIdPartition>
removedPartitions) {
- log.info("Updating assignments for addedPartitions: {} and
removedPartition: {}", addedPartitions, removedPartitions);
+ private void updateAssignments(final Set<TopicIdPartition> addedPartitions,
+ final Set<TopicIdPartition>
removedPartitions) {
+ log.info("Updating assignments for partitions added: {} and removed:
{}", addedPartitions, removedPartitions);
+ if (!addedPartitions.isEmpty() || !removedPartitions.isEmpty()) {
+ synchronized (assignPartitionsLock) {
+ // Make a copy of the existing assignments and update the copy.
+ final Map<TopicIdPartition, UserTopicIdPartition>
updatedUserPartitions = new HashMap<>(assignedUserTopicIdPartitions);
+ addedPartitions.forEach(tpId ->
updatedUserPartitions.putIfAbsent(tpId, newUserTopicIdPartition(tpId)));
+ removedPartitions.forEach(updatedUserPartitions::remove);
+ if
(!updatedUserPartitions.equals(assignedUserTopicIdPartitions)) {
+ assignedUserTopicIdPartitions =
Collections.unmodifiableMap(updatedUserPartitions);
+ hasAssignmentChanged = true;
+ log.debug("Assigned user-topic-partitions: {}",
assignedUserTopicIdPartitions);
+ assignPartitionsLock.notifyAll();
+ }
+ }
+ }
+ }
- Objects.requireNonNull(addedPartitions, "addedPartitions must not be
null");
- Objects.requireNonNull(removedPartitions, "removedPartitions must not
be null");
+ public Optional<Long> receivedOffsetForPartition(final int partition) {
Review Comment:
nit: reuse latest concept (and all usage methods called the same)
```suggestion
public Optional<Long> latestOffsetForMetadataPartition(final int
partition) {
```
##########
storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java:
##########
@@ -64,302 +64,395 @@
class ConsumerTask implements Runnable, Closeable {
private static final Logger log =
LoggerFactory.getLogger(ConsumerTask.class);
- private static final long POLL_INTERVAL_MS = 100L;
+ static long pollIntervalMs = 100L;
private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
- private final KafkaConsumer<byte[], byte[]> consumer;
- private final String metadataTopicName;
+ private final Consumer<byte[], byte[]> consumer;
private final RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler;
private final RemoteLogMetadataTopicPartitioner topicPartitioner;
private final Time time;
- // It indicates whether the closing process has been started or not. If it
is set as true,
- // consumer will stop consuming messages, and it will not allow partition
assignments to be updated.
- private volatile boolean closing = false;
-
- // It indicates whether the consumer needs to assign the partitions or
not. This is set when it is
- // determined that the consumer needs to be assigned with the updated
partitions.
- private volatile boolean assignPartitions = false;
+ // It indicates whether the ConsumerTask is closed or not.
+ private volatile boolean isClosed = false;
+ // It indicates whether the user topic partition assignment to the
consumer has changed or not. If the assignment
+ // has changed, the consumer will eventually start tracking the newly
assigned partitions and stop tracking the
+ // ones it is no longer assigned to.
+ private volatile boolean hasAssignmentChanged = true;
// It represents a lock for any operations related to the
assignedTopicPartitions.
private final Object assignPartitionsLock = new Object();
// Remote log metadata topic partitions that consumer is assigned to.
- private volatile Set<Integer> assignedMetaPartitions =
Collections.emptySet();
+ private volatile Set<Integer> assignedMetadataPartitions =
Collections.emptySet();
// User topic partitions that this broker is a leader/follower for.
- private Set<TopicIdPartition> assignedTopicPartitions =
Collections.emptySet();
+ private volatile Map<TopicIdPartition, UserTopicIdPartition>
assignedUserTopicIdPartitions = Collections.emptyMap();
+ private volatile Set<TopicIdPartition>
processedAssignmentOfUserTopicIdPartitions = Collections.emptySet();
- // Map of remote log metadata topic partition to consumed offsets.
Received consumer records
- // may or may not have been processed based on the assigned topic
partitions.
- private final Map<Integer, Long> partitionToConsumedOffsets = new
ConcurrentHashMap<>();
+ private long uninitializedAt;
+ private boolean isAllUserTopicPartitionsInitialized;
- // Map of remote log metadata topic partition to processed offsets that
were synced in committedOffsetsFile.
- private Map<Integer, Long> lastSyncedPartitionToConsumedOffsets =
Collections.emptyMap();
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> readOffsetsByMetadataPartition = new
ConcurrentHashMap<>();
+ private final Map<TopicIdPartition, Long> readOffsetsByUserTopicPartition
= new HashMap<>();
- private final long committedOffsetSyncIntervalMs;
- private CommittedOffsetsFile committedOffsetsFile;
- private long lastSyncedTimeMs;
+ private Map<TopicPartition, StartAndEndOffsetHolder>
offsetHolderByMetadataPartition = new HashMap<>();
+ private boolean isOffsetsFetchFailed = false;
+ private long lastFailedFetchOffsetsTimestamp;
- public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
- String metadataTopicName,
- RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
+ public ConsumerTask(RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
RemoteLogMetadataTopicPartitioner topicPartitioner,
- Path committedOffsetsPath,
- Time time,
- long committedOffsetSyncIntervalMs) {
- this.consumer = Objects.requireNonNull(consumer);
- this.metadataTopicName = Objects.requireNonNull(metadataTopicName);
+ Consumer<byte[], byte[]> consumer) {
+ this.consumer = consumer;
this.remotePartitionMetadataEventHandler =
Objects.requireNonNull(remotePartitionMetadataEventHandler);
this.topicPartitioner = Objects.requireNonNull(topicPartitioner);
- this.time = Objects.requireNonNull(time);
- this.committedOffsetSyncIntervalMs = committedOffsetSyncIntervalMs;
-
- initializeConsumerAssignment(committedOffsetsPath);
- }
-
- private void initializeConsumerAssignment(Path committedOffsetsPath) {
- try {
- committedOffsetsFile = new
CommittedOffsetsFile(committedOffsetsPath.toFile());
- } catch (IOException e) {
- throw new KafkaException(e);
- }
-
- Map<Integer, Long> committedOffsets = Collections.emptyMap();
- try {
- // Load committed offset and assign them in the consumer.
- committedOffsets = committedOffsetsFile.readEntries();
- } catch (IOException e) {
- // Ignore the error and consumer consumes from the earliest offset.
- log.error("Encountered error while building committed offsets from
the file. " +
- "Consumer will consume from the earliest offset
for the assigned partitions.", e);
- }
-
- if (!committedOffsets.isEmpty()) {
- // Assign topic partitions from the earlier committed offsets file.
- Set<Integer> earlierAssignedPartitions = committedOffsets.keySet();
- assignedMetaPartitions =
Collections.unmodifiableSet(earlierAssignedPartitions);
- Set<TopicPartition> metadataTopicPartitions =
earlierAssignedPartitions.stream()
-
.map(x -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, x))
-
.collect(Collectors.toSet());
- consumer.assign(metadataTopicPartitions);
-
- // Seek to the committed offsets
- for (Map.Entry<Integer, Long> entry : committedOffsets.entrySet())
{
- log.debug("Updating consumed offset: [{}] for partition [{}]",
entry.getValue(), entry.getKey());
- partitionToConsumedOffsets.put(entry.getKey(),
entry.getValue());
- consumer.seek(new
TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, entry.getKey()),
entry.getValue());
- }
-
- lastSyncedPartitionToConsumedOffsets =
Collections.unmodifiableMap(committedOffsets);
- }
+ this.time = new SystemTime();
+ this.uninitializedAt = time.milliseconds();
}
@Override
public void run() {
- log.info("Started Consumer task thread.");
- lastSyncedTimeMs = time.milliseconds();
- try {
- while (!closing) {
- maybeWaitForPartitionsAssignment();
+ log.info("Starting consumer task thread.");
+ while (!isClosed) {
+ try {
+ if (hasAssignmentChanged) {
+ maybeWaitForPartitionsAssignment();
+ }
log.trace("Polling consumer to receive remote log metadata
topic records");
- ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(POLL_INTERVAL_MS));
+ final ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(pollIntervalMs));
for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
processConsumerRecord(record);
}
-
- maybeSyncCommittedDataAndOffsets(false);
+ maybeMarkUserPartitionsAsReady();
+ } catch (final WakeupException ex) {
+ // ignore logging the error
+ isClosed = true;
+ } catch (final RetriableException ex) {
+ log.warn("Retriable error occurred while processing the
records. Retrying...", ex);
+ } catch (final Exception ex) {
+ isClosed = true;
+ log.error("Error occurred while processing the records", ex);
}
- } catch (Exception e) {
- log.error("Error occurred in consumer task, close:[{}]", closing,
e);
- } finally {
- maybeSyncCommittedDataAndOffsets(true);
- closeConsumer();
- log.info("Exiting from consumer task thread");
}
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (final Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ log.info("Exited from consumer task thread");
}
private void processConsumerRecord(ConsumerRecord<byte[], byte[]> record) {
- // Taking assignPartitionsLock here as updateAssignmentsForPartitions
changes assignedTopicPartitions
- // and also calls
remotePartitionMetadataEventHandler.clearTopicPartition(removedPartition) for
the removed
- // partitions.
- RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
- synchronized (assignPartitionsLock) {
- if
(assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
-
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
- } else {
- log.debug("This event {} is skipped as the topic partition is
not assigned for this instance.", remoteLogMetadata);
- }
- log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
- partitionToConsumedOffsets.put(record.partition(),
record.offset());
+ final RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
+ if (canProcess(remoteLogMetadata, record.offset())) {
+
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+
readOffsetsByUserTopicPartition.put(remoteLogMetadata.topicIdPartition(),
record.offset());
+ } else {
+ log.debug("The event {} is skipped because it is either already
processed or not assigned to this consumer", remoteLogMetadata);
}
+ log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
+ readOffsetsByMetadataPartition.put(record.partition(),
record.offset());
+ }
+
+ private boolean canProcess(final RemoteLogMetadata metadata, final long
recordOffset) {
+ final TopicIdPartition tpId = metadata.topicIdPartition();
+ final Long readOffset = readOffsetsByUserTopicPartition.get(tpId);
+ return processedAssignmentOfUserTopicIdPartitions.contains(tpId) &&
(readOffset == null || readOffset < recordOffset);
}
Review Comment:
nit:
```suggestion
private boolean shouldProcess(final RemoteLogMetadata metadata, final
long recordOffset) {
final TopicIdPartition tpId = metadata.topicIdPartition();
final Long latestOffset =
latestOffsetsByUserTopicPartition.getOrDefault(tpId, -1L);
return processedUserTopicIdPartitions.contains(tpId) &&
(latestOffset < recordOffset);
}
```
##########
storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java:
##########
@@ -64,302 +65,393 @@
class ConsumerTask implements Runnable, Closeable {
private static final Logger log =
LoggerFactory.getLogger(ConsumerTask.class);
- private static final long POLL_INTERVAL_MS = 100L;
+ static long pollIntervalMs = 100L;
Review Comment:
Got it. Though, I'd argue a mutable static field may not be the best option
here. Adding `pollIntervalMs` as input parameter/config may be better?
Also, isn't 100ms too often to poll again? Maybe a few seconds or higher may
be a better default.
##########
storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java:
##########
@@ -64,302 +64,395 @@
class ConsumerTask implements Runnable, Closeable {
private static final Logger log =
LoggerFactory.getLogger(ConsumerTask.class);
- private static final long POLL_INTERVAL_MS = 100L;
+ static long pollIntervalMs = 100L;
private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
- private final KafkaConsumer<byte[], byte[]> consumer;
- private final String metadataTopicName;
+ private final Consumer<byte[], byte[]> consumer;
private final RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler;
private final RemoteLogMetadataTopicPartitioner topicPartitioner;
private final Time time;
- // It indicates whether the closing process has been started or not. If it
is set as true,
- // consumer will stop consuming messages, and it will not allow partition
assignments to be updated.
- private volatile boolean closing = false;
-
- // It indicates whether the consumer needs to assign the partitions or
not. This is set when it is
- // determined that the consumer needs to be assigned with the updated
partitions.
- private volatile boolean assignPartitions = false;
+ // It indicates whether the ConsumerTask is closed or not.
+ private volatile boolean isClosed = false;
+ // It indicates whether the user topic partition assignment to the
consumer has changed or not. If the assignment
+ // has changed, the consumer will eventually start tracking the newly
assigned partitions and stop tracking the
+ // ones it is no longer assigned to.
+ private volatile boolean hasAssignmentChanged = true;
// It represents a lock for any operations related to the
assignedTopicPartitions.
private final Object assignPartitionsLock = new Object();
// Remote log metadata topic partitions that consumer is assigned to.
- private volatile Set<Integer> assignedMetaPartitions =
Collections.emptySet();
+ private volatile Set<Integer> assignedMetadataPartitions =
Collections.emptySet();
// User topic partitions that this broker is a leader/follower for.
- private Set<TopicIdPartition> assignedTopicPartitions =
Collections.emptySet();
+ private volatile Map<TopicIdPartition, UserTopicIdPartition>
assignedUserTopicIdPartitions = Collections.emptyMap();
+ private volatile Set<TopicIdPartition>
processedAssignmentOfUserTopicIdPartitions = Collections.emptySet();
- // Map of remote log metadata topic partition to consumed offsets.
Received consumer records
- // may or may not have been processed based on the assigned topic
partitions.
- private final Map<Integer, Long> partitionToConsumedOffsets = new
ConcurrentHashMap<>();
+ private long uninitializedAt;
+ private boolean isAllUserTopicPartitionsInitialized;
- // Map of remote log metadata topic partition to processed offsets that
were synced in committedOffsetsFile.
- private Map<Integer, Long> lastSyncedPartitionToConsumedOffsets =
Collections.emptyMap();
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> readOffsetsByMetadataPartition = new
ConcurrentHashMap<>();
+ private final Map<TopicIdPartition, Long> readOffsetsByUserTopicPartition
= new HashMap<>();
- private final long committedOffsetSyncIntervalMs;
- private CommittedOffsetsFile committedOffsetsFile;
- private long lastSyncedTimeMs;
+ private Map<TopicPartition, StartAndEndOffsetHolder>
offsetHolderByMetadataPartition = new HashMap<>();
+ private boolean isOffsetsFetchFailed = false;
+ private long lastFailedFetchOffsetsTimestamp;
- public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
- String metadataTopicName,
- RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
+ public ConsumerTask(RemotePartitionMetadataEventHandler
remotePartitionMetadataEventHandler,
RemoteLogMetadataTopicPartitioner topicPartitioner,
- Path committedOffsetsPath,
- Time time,
- long committedOffsetSyncIntervalMs) {
- this.consumer = Objects.requireNonNull(consumer);
- this.metadataTopicName = Objects.requireNonNull(metadataTopicName);
+ Consumer<byte[], byte[]> consumer) {
+ this.consumer = consumer;
this.remotePartitionMetadataEventHandler =
Objects.requireNonNull(remotePartitionMetadataEventHandler);
this.topicPartitioner = Objects.requireNonNull(topicPartitioner);
- this.time = Objects.requireNonNull(time);
- this.committedOffsetSyncIntervalMs = committedOffsetSyncIntervalMs;
-
- initializeConsumerAssignment(committedOffsetsPath);
- }
-
- private void initializeConsumerAssignment(Path committedOffsetsPath) {
- try {
- committedOffsetsFile = new
CommittedOffsetsFile(committedOffsetsPath.toFile());
- } catch (IOException e) {
- throw new KafkaException(e);
- }
-
- Map<Integer, Long> committedOffsets = Collections.emptyMap();
- try {
- // Load committed offset and assign them in the consumer.
- committedOffsets = committedOffsetsFile.readEntries();
- } catch (IOException e) {
- // Ignore the error and consumer consumes from the earliest offset.
- log.error("Encountered error while building committed offsets from
the file. " +
- "Consumer will consume from the earliest offset
for the assigned partitions.", e);
- }
-
- if (!committedOffsets.isEmpty()) {
- // Assign topic partitions from the earlier committed offsets file.
- Set<Integer> earlierAssignedPartitions = committedOffsets.keySet();
- assignedMetaPartitions =
Collections.unmodifiableSet(earlierAssignedPartitions);
- Set<TopicPartition> metadataTopicPartitions =
earlierAssignedPartitions.stream()
-
.map(x -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, x))
-
.collect(Collectors.toSet());
- consumer.assign(metadataTopicPartitions);
-
- // Seek to the committed offsets
- for (Map.Entry<Integer, Long> entry : committedOffsets.entrySet())
{
- log.debug("Updating consumed offset: [{}] for partition [{}]",
entry.getValue(), entry.getKey());
- partitionToConsumedOffsets.put(entry.getKey(),
entry.getValue());
- consumer.seek(new
TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, entry.getKey()),
entry.getValue());
- }
-
- lastSyncedPartitionToConsumedOffsets =
Collections.unmodifiableMap(committedOffsets);
- }
+ this.time = new SystemTime();
+ this.uninitializedAt = time.milliseconds();
}
@Override
public void run() {
- log.info("Started Consumer task thread.");
- lastSyncedTimeMs = time.milliseconds();
- try {
- while (!closing) {
- maybeWaitForPartitionsAssignment();
+ log.info("Starting consumer task thread.");
+ while (!isClosed) {
+ try {
+ if (hasAssignmentChanged) {
+ maybeWaitForPartitionsAssignment();
+ }
log.trace("Polling consumer to receive remote log metadata
topic records");
- ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(POLL_INTERVAL_MS));
+ final ConsumerRecords<byte[], byte[]> consumerRecords =
consumer.poll(Duration.ofMillis(pollIntervalMs));
for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
processConsumerRecord(record);
}
-
- maybeSyncCommittedDataAndOffsets(false);
+ maybeMarkUserPartitionsAsReady();
+ } catch (final WakeupException ex) {
+ // ignore logging the error
+ isClosed = true;
+ } catch (final RetriableException ex) {
+ log.warn("Retriable error occurred while processing the
records. Retrying...", ex);
+ } catch (final Exception ex) {
+ isClosed = true;
+ log.error("Error occurred while processing the records", ex);
}
- } catch (Exception e) {
- log.error("Error occurred in consumer task, close:[{}]", closing,
e);
- } finally {
- maybeSyncCommittedDataAndOffsets(true);
- closeConsumer();
- log.info("Exiting from consumer task thread");
}
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (final Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ log.info("Exited from consumer task thread");
}
private void processConsumerRecord(ConsumerRecord<byte[], byte[]> record) {
- // Taking assignPartitionsLock here as updateAssignmentsForPartitions
changes assignedTopicPartitions
- // and also calls
remotePartitionMetadataEventHandler.clearTopicPartition(removedPartition) for
the removed
- // partitions.
- RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
- synchronized (assignPartitionsLock) {
- if
(assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
-
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
- } else {
- log.debug("This event {} is skipped as the topic partition is
not assigned for this instance.", remoteLogMetadata);
- }
- log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
- partitionToConsumedOffsets.put(record.partition(),
record.offset());
+ final RemoteLogMetadata remoteLogMetadata =
serde.deserialize(record.value());
+ if (canProcess(remoteLogMetadata, record.offset())) {
+
remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+
readOffsetsByUserTopicPartition.put(remoteLogMetadata.topicIdPartition(),
record.offset());
+ } else {
+ log.debug("The event {} is skipped because it is either already
processed or not assigned to this consumer", remoteLogMetadata);
}
+ log.debug("Updating consumed offset: [{}] for partition [{}]",
record.offset(), record.partition());
+ readOffsetsByMetadataPartition.put(record.partition(),
record.offset());
+ }
+
+ private boolean canProcess(final RemoteLogMetadata metadata, final long
recordOffset) {
+ final TopicIdPartition tpId = metadata.topicIdPartition();
+ final Long readOffset = readOffsetsByUserTopicPartition.get(tpId);
+ return processedAssignmentOfUserTopicIdPartitions.contains(tpId) &&
(readOffset == null || readOffset < recordOffset);
}
- private void maybeSyncCommittedDataAndOffsets(boolean forceSync) {
- // Return immediately if there is no consumption from last time.
- boolean noConsumedOffsetUpdates =
partitionToConsumedOffsets.equals(lastSyncedPartitionToConsumedOffsets);
- if (noConsumedOffsetUpdates || !forceSync && time.milliseconds() -
lastSyncedTimeMs < committedOffsetSyncIntervalMs) {
- log.debug("Skip syncing committed offsets,
noConsumedOffsetUpdates: {}, forceSync: {}", noConsumedOffsetUpdates,
forceSync);
+ private void maybeMarkUserPartitionsAsReady() {
+ if (isAllUserTopicPartitionsInitialized) {
return;
}
-
- try {
- // Need to take lock on assignPartitionsLock as
assignedTopicPartitions might
- // get updated by other threads.
- synchronized (assignPartitionsLock) {
- for (TopicIdPartition topicIdPartition :
assignedTopicPartitions) {
- int metadataPartition =
topicPartitioner.metadataPartition(topicIdPartition);
- Long offset =
partitionToConsumedOffsets.get(metadataPartition);
- if (offset != null) {
-
remotePartitionMetadataEventHandler.syncLogMetadataSnapshot(topicIdPartition,
metadataPartition, offset);
+ maybeFetchStartAndEndOffsets();
+ boolean isAllInitialized = true;
+ for (final UserTopicIdPartition utp :
assignedUserTopicIdPartitions.values()) {
+ if (utp.isAssigned && !utp.isInitialized) {
+ final Integer metadataPartition = utp.metadataPartition;
+ final StartAndEndOffsetHolder holder =
offsetHolderByMetadataPartition.get(toRemoteLogPartition(metadataPartition));
+ // The offset-holder can be null, when the recent assignment
wasn't picked up by the consumer.
+ if (holder != null) {
+ final Long readOffset =
readOffsetsByMetadataPartition.getOrDefault(metadataPartition, -1L);
+ // 1) The end-offset was fetched only once during
reassignment. The metadata-partition can receive
+ // new stream of records, so the consumer can read records
more than the last-fetched end-offset.
+ // 2) When the internal topic becomes empty due to breach
by size/time/start-offset, then there
+ // are no records to read.
+ if (readOffset + 1 >= holder.endOffset ||
holder.endOffset.equals(holder.startOffset)) {
+ markInitialized(utp);
} else {
- log.debug("Skipping sync-up of the
remote-log-metadata-file for partition: [{}] , with remote log metadata
partition{}, and no offset",
- topicIdPartition, metadataPartition);
+ log.debug("The user-topic-partition {} could not be
marked initialized since the read-offset is {} " +
+ "but the end-offset is {} for the
metadata-partition {}", utp, readOffset, holder.endOffset,
+ metadataPartition);
}
+ } else {
+ log.debug("The offset-holder is null for the
metadata-partition {}. The consumer may not have picked" +
+ " up the recent assignment", metadataPartition);
}
-
- // Write partitionToConsumedOffsets into committed offsets
file as we do not want to process them again
- // in case of restarts.
- committedOffsetsFile.writeEntries(partitionToConsumedOffsets);
- lastSyncedPartitionToConsumedOffsets = new
HashMap<>(partitionToConsumedOffsets);
}
-
- lastSyncedTimeMs = time.milliseconds();
- } catch (IOException e) {
- throw new KafkaException("Error encountered while writing
committed offsets to a local file", e);
+ isAllInitialized = isAllInitialized && utp.isInitialized;
}
- }
-
- private void closeConsumer() {
- log.info("Closing the consumer instance");
- try {
- consumer.close(Duration.ofSeconds(30));
- } catch (Exception e) {
- log.error("Error encountered while closing the consumer", e);
+ if (isAllInitialized) {
+ log.info("Initialized for all the {} assigned user-partitions
mapped to the {} meta-partitions in {} ms",
+ assignedUserTopicIdPartitions.size(),
assignedMetadataPartitions.size(),
+ time.milliseconds() - uninitializedAt);
}
+ isAllUserTopicPartitionsInitialized = isAllInitialized;
}
- private void maybeWaitForPartitionsAssignment() {
- Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ void maybeWaitForPartitionsAssignment() throws InterruptedException {
Review Comment:
nit:
```suggestion
void maybeWaitForPartitionAssignments() throws InterruptedException {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
