hachikuji commented on code in PR #15671: URL: https://github.com/apache/kafka/pull/15671#discussion_r1576879728
########## clients/src/main/java/org/apache/kafka/common/feature/BaseVersionRange.java: ########## @@ -26,7 +26,7 @@ /** * Represents an immutable basic version range using 2 attributes: min and max, each of type short. * The min and max attributes need to satisfy 2 rules: - * - they are each expected to be >= 1, as we only consider positive version values to be valid. Review Comment: Was it a bug that we only allowed version 1 and above? I'm wondering if we really need to change it. ########## clients/src/main/resources/common/message/VotersRecord.json: ########## @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "type": "data", + "name": "VotersRecord", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "Version", "type": "int16", "versions": "0+", + "about": "The version of the voters record" }, + { "name": "Voters", "type": "[]Voter", "versions": "0+", "fields": [ + { "name": "VoterId", "type": "int32", "versions": "0+", "entityType": "brokerId", + "about": "The replica id of the voter in the topic partition" }, + { "name": "VoterUuid", "type": "uuid", "versions": "0+", Review Comment: Why don't we call it `VoterDirectoryId`? ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); + } + + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.put(voter.id(), voter); + + return Optional.of(new VoterSet(newVoters)); + } + + /** + * Removew a voter from the voter set. + * + * This object is immutable. A new voter set is returned if the voter was removed. + * + * A voter can be removed from the voter set if its id and uuid match. + * + * @param voterId the voter id + * @param voterUuid the voter uuid + * @return a new voter set if the voter was remove, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> removeVoter(int voterId, Optional<Uuid> voterUuid) { + VoterNode oldVoter = voters.get(voterId); + if (oldVoter != null && Objects.equals(oldVoter.uuid(), voterUuid)) { + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.remove(voterId); + + return Optional.of(new VoterSet(newVoters)); + } + + return Optional.empty(); + } + + /** + * Converts a voter set to a voters record for a given version. + * + * @param version the version of the voters record + */ + public VotersRecord toVotersRecord(short version) { + return new VotersRecord() + .setVersion(version) + .setVoters( + voters + .values() + .stream() + .map(voter -> { + Iterator<VotersRecord.Endpoint> endpoints = voter + .listeners() + .entrySet() + .stream() + .map(entry -> + new VotersRecord.Endpoint() + .setName(entry.getKey()) + .setHost(entry.getValue().getHostString()) + .setPort(entry.getValue().getPort()) + ) + .iterator(); + + VotersRecord.KRaftVersionFeature kraftVersionFeature = new VotersRecord.KRaftVersionFeature() + .setMinSupportedVersion(voter.supportedKRaftVersion().min()) + .setMaxSupportedVersion(voter.supportedKRaftVersion().max()); + + return new VotersRecord.Voter() + .setVoterId(voter.id()) + .setVoterUuid(voter.uuid().orElse(Uuid.ZERO_UUID)) + .setEndpoints(new VotersRecord.EndpointCollection(endpoints)) + .setKRaftVersionFeature(kraftVersionFeature); + }) + .collect(Collectors.toList()) + ); + } + + /** + * Determines if two sets of voters have an overlapping majority. Review Comment: It would be helpful to clarify that we only consider voter ID to determine an overlapping majority. ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { Review Comment: The generic naming of this might cause confusion. Perhaps it could be `KraftControlRecordListener` or something like that? ########## core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala: ########## @@ -324,15 +323,11 @@ class DumpLogSegmentsTest { val lastContainedLogTimestamp = 10000 TestUtils.resource( - RecordsSnapshotWriter.createWithHeader( - () => metadataLog.createNewSnapshot(new OffsetAndEpoch(0, 0)), - 1024, - MemoryPool.NONE, - new MockTime, - lastContainedLogTimestamp, - CompressionType.NONE, - MetadataRecordSerde.INSTANCE, - ).get() + new RecordsSnapshotWriter.Builder() Review Comment: Do we need any assertions for the new control records? ########## raft/src/main/java/org/apache/kafka/raft/internals/TreeMapHistory.java: ########## @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; +import java.util.Map; + +/** + * A implementation for {@code History} which uses a red-black tree to store values sorted by offset. + */ +final public class TreeMapHistory<T> implements History<T> { + private final NavigableMap<Long, T> history = new TreeMap<>(); + + @Override + public void addAt(long offset, T value) { + if (offset < 0) { + throw new IllegalArgumentException( + String.format("Next offset %d must be greater than or equal to 0", offset) + ); + } + + Map.Entry<Long, ?> lastEntry = history.lastEntry(); Review Comment: nit: could we just use `this.lastEntry`? ########## clients/src/main/resources/common/message/VotersRecord.json: ########## @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "type": "data", + "name": "VotersRecord", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "Version", "type": "int16", "versions": "0+", + "about": "The version of the voters record" }, + { "name": "Voters", "type": "[]Voter", "versions": "0+", "fields": [ + { "name": "VoterId", "type": "int32", "versions": "0+", "entityType": "brokerId", + "about": "The replica id of the voter in the topic partition" }, + { "name": "VoterUuid", "type": "uuid", "versions": "0+", + "about": "The directory id of the voter in the topic partition" }, + { "name": "Endpoints", "type": "[]Endpoint", "versions": "0+", + "about": "The endpoint that can be used to communicate with the voter", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The name of the endpoint" }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The hostname" }, + { "name": "Port", "type": "uint16", "versions": "0+", Review Comment: It's interesting that we don't indicate the security protocol. I wonder if there is an argument for doing so. Perhaps it could be an easy way to detect misconfigurations? ########## clients/src/test/java/org/apache/kafka/common/record/ControlRecordTypeTest.java: ########## @@ -45,4 +45,58 @@ public void testParseUnknownVersion() { assertEquals(ControlRecordType.ABORT, type); } + @Test + public void testLeaderChange() { + ByteBuffer buffer = ByteBuffer.allocate(32); + buffer.putShort(ControlRecordType.CURRENT_CONTROL_RECORD_KEY_VERSION); + buffer.putShort((short) 2); + buffer.flip(); + + ControlRecordType type = ControlRecordType.parse(buffer); + assertEquals(ControlRecordType.LEADER_CHANGE, type); + } + + @Test + public void testSnapshotHeader() { + ByteBuffer buffer = ByteBuffer.allocate(32); + buffer.putShort(ControlRecordType.CURRENT_CONTROL_RECORD_KEY_VERSION); + buffer.putShort((short) 3); + buffer.flip(); + + ControlRecordType type = ControlRecordType.parse(buffer); + assertEquals(ControlRecordType.SNAPSHOT_HEADER, type); + } + + @Test + public void testSnapshotFooter() { Review Comment: nit: These tests seem mostly the same. Would it be possible to use a parameterized test instead? ########## raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java: ########## @@ -261,7 +260,7 @@ default long truncateToEndOffset(OffsetAndEpoch endOffset) { * @param snapshotId the end offset and epoch that identifies the snapshot * @return a writable snapshot if it doesn't already exist */ - Optional<RawSnapshotWriter> storeSnapshot(OffsetAndEpoch snapshotId); + Optional<RawSnapshotWriter> createNewSnapshotUnchecked(OffsetAndEpoch snapshotId); Review Comment: In what sense is this "unchecked"? It would be helpful to clarify the in the documentation. ########## core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala: ########## @@ -1341,29 +1340,25 @@ class KafkaConfigTest { } @Test - def testValidQuorumVotersConfig(): Unit = { - val expected = new util.HashMap[Integer, AddressSpec]() + def testValidQuorumVotersParsing(): Unit = { + val expected = new util.HashMap[Integer, InetSocketAddress]() Review Comment: nit: the use of `expected` seems a little clumsy. Could we just separate the two cases into two tests? ########## clients/src/main/java/org/apache/kafka/common/record/ControlRecordType.java: ########## @@ -44,11 +44,15 @@ public enum ControlRecordType { ABORT((short) 0), COMMIT((short) 1), - // Raft quorum related control messages. + // KRaft quorum related control messages LEADER_CHANGE((short) 2), SNAPSHOT_HEADER((short) 3), SNAPSHOT_FOOTER((short) 4), + // KRaft membership changes messages + KRAFT_VERSION((short) 5), + VOTERS((short) 6), Review Comment: nit: maybe we can start using the prefix consistently. `KRAFT_VOTERS`? ########## clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java: ########## @@ -807,4 +809,62 @@ private static void writeSnapshotFooterRecord( builder.appendSnapshotFooterMessage(timestamp, snapshotFooterRecord); } } + + public static MemoryRecords withKRaftVersionRecord( + long initialOffset, + long timestamp, + int leaderEpoch, + ByteBuffer buffer, + KRaftVersionRecord kraftVersionRecord + ) { + writeKRaftVersionRecord(buffer, initialOffset, timestamp, leaderEpoch, kraftVersionRecord); + buffer.flip(); + return MemoryRecords.readableRecords(buffer); + } + + private static void writeKRaftVersionRecord( + ByteBuffer buffer, + long initialOffset, + long timestamp, + int leaderEpoch, + KRaftVersionRecord kraftVersionRecord + ) { + try (MemoryRecordsBuilder builder = new MemoryRecordsBuilder( + buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, initialOffset, timestamp, + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, + false, true, leaderEpoch, buffer.capacity()) + ) { + builder.appendKRaftVersionMessage(timestamp, kraftVersionRecord); + } + } + + public static MemoryRecords withVotersRecord( + long initialOffset, + long timestamp, + int leaderEpoch, + ByteBuffer buffer, + VotersRecord votersRecord + ) { + writeVotersRecord(buffer, initialOffset, timestamp, leaderEpoch, votersRecord); + buffer.flip(); + return MemoryRecords.readableRecords(buffer); + } + + private static void writeVotersRecord( + ByteBuffer buffer, + long initialOffset, + long timestamp, + int leaderEpoch, + VotersRecord votersRecord + ) { + try (MemoryRecordsBuilder builder = new MemoryRecordsBuilder( + buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, initialOffset, timestamp, Review Comment: nit: as long as we're separating args into lines, how about one argument per line? ########## core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala: ########## @@ -87,9 +85,12 @@ class RaftClusterSnapshotTest { // Check that we can read the entire snapshot while (snapshot.hasNext) { - val batch = snapshot.next() + val batch = snapshot.next assertTrue(batch.sizeInBytes > 0) - assertNotEquals(Collections.emptyList(), batch.records()) + assertTrue( + batch.records.isEmpty != batch.controlRecords.isEmpty, Review Comment: nit: this assertion is not very intuitive. I wonder if it would help to store the result in a nicely named variable. ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} Review Comment: nit: exist**s** ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); + } + + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.put(voter.id(), voter); + + return Optional.of(new VoterSet(newVoters)); + } + + /** + * Removew a voter from the voter set. Review Comment: nit: Remove ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); + } + + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.put(voter.id(), voter); + + return Optional.of(new VoterSet(newVoters)); + } + + /** + * Removew a voter from the voter set. + * + * This object is immutable. A new voter set is returned if the voter was removed. + * + * A voter can be removed from the voter set if its id and uuid match. + * + * @param voterId the voter id + * @param voterUuid the voter uuid + * @return a new voter set if the voter was remove, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> removeVoter(int voterId, Optional<Uuid> voterUuid) { + VoterNode oldVoter = voters.get(voterId); + if (oldVoter != null && Objects.equals(oldVoter.uuid(), voterUuid)) { + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.remove(voterId); + + return Optional.of(new VoterSet(newVoters)); + } + + return Optional.empty(); + } + + /** + * Converts a voter set to a voters record for a given version. + * + * @param version the version of the voters record + */ + public VotersRecord toVotersRecord(short version) { + return new VotersRecord() + .setVersion(version) + .setVoters( + voters + .values() + .stream() + .map(voter -> { + Iterator<VotersRecord.Endpoint> endpoints = voter + .listeners() Review Comment: nit: all the nesting here affects readability. Perhaps it would be easier to follow with some helpers or by introducing local variables? ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { Review Comment: Might not be worth it, but I guess we could make offset generic to. I imagine the code is mostly the same if we just had `S extends Comparable`. ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); + } + + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.put(voter.id(), voter); + + return Optional.of(new VoterSet(newVoters)); + } + + /** + * Removew a voter from the voter set. + * + * This object is immutable. A new voter set is returned if the voter was removed. + * + * A voter can be removed from the voter set if its id and uuid match. + * + * @param voterId the voter id + * @param voterUuid the voter uuid + * @return a new voter set if the voter was remove, otherwise {@code Optional.empty()} Review Comment: nit: remove**d** ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); Review Comment: Wondering if we should verify that the `VoterNode` object is the same. ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSet.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.feature.SupportedVersionRange; + +/** + * A type for representing the set of voters for a topic partition. + * + * It encapsulates static information like a voter's endpoint and their supported kraft.version. + * + * It providees functionality for converting to and from {@code VotersRecord} and for converting + * from the static configuration. + */ +final public class VoterSet { + private final Map<Integer, VoterNode> voters; + + VoterSet(Map<Integer, VoterNode> voters) { + if (voters.isEmpty()) { + throw new IllegalArgumentException("Voters cannot be empty"); + } + + this.voters = voters; + } + + /** + * Returns the socket address for a given voter at a given listener. + * + * @param voter the id of the voter + * @param listener the name of the listener + * @return the socket address if it exist, otherwise {@code Optional.empty()} + */ + public Optional<InetSocketAddress> voterAddress(int voter, String listener) { + return Optional.ofNullable(voters.get(voter)) + .flatMap(voterNode -> voterNode.address(listener)); + } + + /** + * Returns all of the voter ids. + */ + public Set<Integer> voterIds() { + return voters.keySet(); + } + + /** + * Adds a voter to the voter set. + * + * This object is immutable. A new voter set is returned if the voter was added. + * + * A new voter can be added to a voter set if its id doesn't already exist in the voter set. + * + * @param voter the new voter to add + * @return a new voter set if the voter was added, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> addVoter(VoterNode voter) { + if (voters.containsKey(voter.id())) { + return Optional.empty(); + } + + HashMap<Integer, VoterNode> newVoters = new HashMap<>(voters); + newVoters.put(voter.id(), voter); + + return Optional.of(new VoterSet(newVoters)); + } + + /** + * Removew a voter from the voter set. + * + * This object is immutable. A new voter set is returned if the voter was removed. + * + * A voter can be removed from the voter set if its id and uuid match. + * + * @param voterId the voter id Review Comment: It would be helpful to clarify the use of this parameter. At a glance, I would have assumed that the id would have been ignored if it was empty (i.e. categorical removal by just voter id). ########## clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java: ########## @@ -807,4 +809,62 @@ private static void writeSnapshotFooterRecord( builder.appendSnapshotFooterMessage(timestamp, snapshotFooterRecord); } } + + public static MemoryRecords withKRaftVersionRecord( + long initialOffset, + long timestamp, + int leaderEpoch, + ByteBuffer buffer, + KRaftVersionRecord kraftVersionRecord + ) { + writeKRaftVersionRecord(buffer, initialOffset, timestamp, leaderEpoch, kraftVersionRecord); + buffer.flip(); + return MemoryRecords.readableRecords(buffer); + } + + private static void writeKRaftVersionRecord( + ByteBuffer buffer, + long initialOffset, + long timestamp, + int leaderEpoch, + KRaftVersionRecord kraftVersionRecord + ) { + try (MemoryRecordsBuilder builder = new MemoryRecordsBuilder( + buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, initialOffset, timestamp, + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, + false, true, leaderEpoch, buffer.capacity()) + ) { + builder.appendKRaftVersionMessage(timestamp, kraftVersionRecord); + } + } + + public static MemoryRecords withVotersRecord( + long initialOffset, + long timestamp, + int leaderEpoch, + ByteBuffer buffer, + VotersRecord votersRecord + ) { + writeVotersRecord(buffer, initialOffset, timestamp, leaderEpoch, votersRecord); Review Comment: nit: This looks a little odd. We create two `MemoryRecords` instances. Why not just create one? ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided + * offset. + * + * @param offset the offset + * @return the value if it exist, otherwise {@code Optional.empty()} Review Comment: nit: exist**s** ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided + * offset. + * + * @param offset the offset + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<T> valueAtOrBefore(long offset); + + /** + * Returns the value with the largest offset. + * + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<Entry<T>> lastEntry(); + + /** + * Removes all entries with an offset greater than or equal to {@code endOffset}. + * + * @param endOffset the ending offset + */ + public void truncateTo(long endOffset); Review Comment: Perhaps `truncateToEndOffset` to avoid ambiguity. ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided Review Comment: nit: less than or equal to ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided + * offset. + * + * @param offset the offset + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<T> valueAtOrBefore(long offset); + + /** + * Returns the value with the largest offset. + * + * @return the value if it exist, otherwise {@code Optional.empty()} Review Comment: nit: exist**s** ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors Review Comment: nit: These objects are synchronized. Do you mean "respective"? ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided + * offset. + * + * @param offset the offset + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<T> valueAtOrBefore(long offset); + + /** + * Returns the value with the largest offset. + * + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<Entry<T>> lastEntry(); + + /** + * Removes all entries with an offset greater than or equal to {@code endOffset}. + * + * @param endOffset the ending offset + */ + public void truncateTo(long endOffset); + + /** + * Removes all entries but the largest entry that has an offset that is less than or equal to + * {@code startOffset}. + * + * This operation does not remove the entry with the largest offset that is less than or equal + * to {@code startOffset}. This is needed so that calls to {@code valueAtOrBefore} and + * {@code lastEntry} always return a non-empty value if a value was previously added to this + * object. + * + * @param startOffset the starting offset + */ + public void trimPrefixTo(long startOffset); Review Comment: Similarly could be `truncateToStartOffset`. Or perhaps `truncateOldEntries` and `truncateNewEntries`? ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. Review Comment: nit: R**e**turns ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> voterSetAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the voter set at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (voterSetHistory) { + return voterSetHistory.valueAtOrBefore(offset); + } + } + + /** + * Returns the finalized kraft version at a given offset. + * + * @param offset the offset (inclusive) + * @return the finalized kraft version if one exist, otherwise 0 + */ + public short kraftVersionAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the kraft.version at an offset (%d) which kraft hasn't seen (%d)", Review Comment: It would be helpful if the comment explained the second parameter. ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. Review Comment: It would be helpful to have some mention of the threading model here (i.e. which threads act interact with this). It would clarify the synchronization below. ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> voterSetAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the voter set at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (voterSetHistory) { + return voterSetHistory.valueAtOrBefore(offset); + } + } + + /** + * Returns the finalized kraft version at a given offset. + * + * @param offset the offset (inclusive) + * @return the finalized kraft version if one exist, otherwise 0 + */ + public short kraftVersionAtOffset(long offset) { Review Comment: nit: it might be possible to use a (private) generic method to avoid the duplication ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> voterSetAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the voter set at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (voterSetHistory) { + return voterSetHistory.valueAtOrBefore(offset); + } + } + + /** + * Returns the finalized kraft version at a given offset. + * + * @param offset the offset (inclusive) + * @return the finalized kraft version if one exist, otherwise 0 + */ + public short kraftVersionAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the kraft.version at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (kraftVersionHistory) { + return kraftVersionHistory.valueAtOrBefore(offset).orElse((short) 0); + } + } + + private void maybeLoadLog() { + while (log.endOffset().offset > nextOffset) { + LogFetchInfo info = log.read(nextOffset, Isolation.UNCOMMITTED); + try (RecordsIterator<?> iterator = new RecordsIterator<>( + info.records, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + while (iterator.hasNext()) { + Batch<?> batch = iterator.next(); + handleBatch(batch, OptionalLong.empty()); + nextOffset = batch.lastOffset() + 1; + } + } + } + } + + private void maybeLoadSnapshot() { + if ((nextOffset == 0 || nextOffset < log.startOffset()) && log.latestSnapshot().isPresent()) { + RawSnapshotReader rawSnapshot = log.latestSnapshot().get(); + // Clear the current state + synchronized (kraftVersionHistory) { + kraftVersionHistory.clear(); + } + synchronized (voterSetHistory) { + voterSetHistory.clear(); + } + + // Load the snapshot since the listener is at the start of the log or the log doesn't have the next entry. + try (SnapshotReader<?> reader = RecordsSnapshotReader.of( + rawSnapshot, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + logger.info( + "Loading snapshot ({}) since log start offset ({}) is greater than the internal listener's next offset ({})", + reader.snapshotId(), + log.startOffset(), + nextOffset + ); + OptionalLong currentOffset = OptionalLong.of(reader.lastContainedLogOffset()); + while (reader.hasNext()) { + Batch<?> batch = reader.next(); Review Comment: Slightly annoying that we load the full batch into memory. It ought to be possible to look at the header only to see if it is a control batch and skip loading otherwise. Potential optimization for the future I guess. ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, Review Comment: It might not matter too much, but we may as well specify `ApiMessage` as the type here. We do have strong type assumptions here. ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> voterSetAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the voter set at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (voterSetHistory) { + return voterSetHistory.valueAtOrBefore(offset); + } + } + + /** + * Returns the finalized kraft version at a given offset. + * + * @param offset the offset (inclusive) + * @return the finalized kraft version if one exist, otherwise 0 + */ + public short kraftVersionAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the kraft.version at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (kraftVersionHistory) { + return kraftVersionHistory.valueAtOrBefore(offset).orElse((short) 0); + } + } + + private void maybeLoadLog() { + while (log.endOffset().offset > nextOffset) { + LogFetchInfo info = log.read(nextOffset, Isolation.UNCOMMITTED); + try (RecordsIterator<?> iterator = new RecordsIterator<>( + info.records, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + while (iterator.hasNext()) { + Batch<?> batch = iterator.next(); + handleBatch(batch, OptionalLong.empty()); + nextOffset = batch.lastOffset() + 1; + } + } + } + } + + private void maybeLoadSnapshot() { + if ((nextOffset == 0 || nextOffset < log.startOffset()) && log.latestSnapshot().isPresent()) { + RawSnapshotReader rawSnapshot = log.latestSnapshot().get(); + // Clear the current state + synchronized (kraftVersionHistory) { + kraftVersionHistory.clear(); + } + synchronized (voterSetHistory) { + voterSetHistory.clear(); + } + + // Load the snapshot since the listener is at the start of the log or the log doesn't have the next entry. + try (SnapshotReader<?> reader = RecordsSnapshotReader.of( + rawSnapshot, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + logger.info( + "Loading snapshot ({}) since log start offset ({}) is greater than the internal listener's next offset ({})", + reader.snapshotId(), + log.startOffset(), + nextOffset + ); + OptionalLong currentOffset = OptionalLong.of(reader.lastContainedLogOffset()); + while (reader.hasNext()) { + Batch<?> batch = reader.next(); + handleBatch(batch, currentOffset); + } + + nextOffset = reader.lastContainedLogOffset() + 1; + } + } + } + + private void handleBatch(Batch<?> batch, OptionalLong overrideOffset) { + int index = 0; + for (ControlRecord record : batch.controlRecords()) { + long currentOffset = overrideOffset.orElse(batch.baseOffset() + index); + switch (record.type()) { + case VOTERS: + synchronized (voterSetHistory) { Review Comment: Should we just make the `History` objects thread-safe? ########## raft/src/main/java/org/apache/kafka/raft/internals/PartitionListener.java: ########## @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.kafka.common.message.KRaftVersionRecord; +import org.apache.kafka.common.message.VotersRecord; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.ControlRecord; +import org.apache.kafka.raft.Isolation; +import org.apache.kafka.raft.LogFetchInfo; +import org.apache.kafka.raft.ReplicatedLog; +import org.apache.kafka.server.common.serialization.RecordSerde; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RecordsSnapshotReader; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +/** + * The KRaft state machine for tracking control records in the topic partition. + * + * This type keeps track of changes to the finalized kraft.version and the sets of voters. + */ +final public class PartitionListener { + private final ReplicatedLog log; + private final RecordSerde<?> serde; + private final BufferSupplier bufferSupplier; + private final Logger logger; + private final int maxBatchSizeBytes; + + // These are objects are synchronized using the perspective object monitor. The two actors + // are the KRaft driver and the RaftClient callers + private final VoterSetHistory voterSetHistory; + private final History<Short> kraftVersionHistory = new TreeMapHistory<>(); + + // This synchronization is enough because + // 1. The write operation updateListener only sets the value without reading and updates to + // voterSetHistory or kraftVersionHistory are done before setting the nextOffset + // + // 2. The read operations lastVoterSet, voterSetAtOffset and kraftVersionAtOffset read + // the nextOffset first before reading voterSetHistory or kraftVersionHistory + private volatile long nextOffset = 0; + + /** + * Constructs an internal log listener + * + * @param staticVoterSet the set of voter statically configured + * @param log the on disk topic partition + * @param serde the record decoder for data records + * @param bufferSupplier the supplier of byte buffers + * @param maxBatchSizeBytes the maximum size of record batch + * @param logContext the log context + */ + public PartitionListener( + Optional<VoterSet> staticVoterSet, + ReplicatedLog log, + RecordSerde<?> serde, + BufferSupplier bufferSupplier, + int maxBatchSizeBytes, + LogContext logContext + ) { + this.log = log; + this.voterSetHistory = new VoterSetHistory(staticVoterSet); + this.serde = serde; + this.bufferSupplier = bufferSupplier; + this.maxBatchSizeBytes = maxBatchSizeBytes; + this.logger = logContext.logger(this.getClass()); + } + + /** + * Must be called whenever the {@code log} has changed. + */ + public void updateListener() { + maybeLoadSnapshot(); + maybeLoadLog(); + } + + /** + * Remove the head of the log until the given offset. + * + * @param endOffset the end offset (exclusive) + */ + public void truncateTo(long endOffset) { + synchronized (voterSetHistory) { + voterSetHistory.truncateTo(endOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.truncateTo(endOffset); + } + } + + /** + * Remove the tail of the log until the given offset. + * + * @param @startOffset the start offset (inclusive) + */ + public void trimPrefixTo(long startOffset) { + synchronized (voterSetHistory) { + voterSetHistory.trimPrefixTo(startOffset); + } + synchronized (kraftVersionHistory) { + kraftVersionHistory.trimPrefixTo(startOffset); + } + } + + /** + * Returns the last voter set. + */ + public VoterSet lastVoterSet() { + synchronized (voterSetHistory) { + return voterSetHistory.lastValue(); + } + } + + /** + * Rturns the voter set at a given offset. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + public Optional<VoterSet> voterSetAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the voter set at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (voterSetHistory) { + return voterSetHistory.valueAtOrBefore(offset); + } + } + + /** + * Returns the finalized kraft version at a given offset. + * + * @param offset the offset (inclusive) + * @return the finalized kraft version if one exist, otherwise 0 + */ + public short kraftVersionAtOffset(long offset) { + long fixedNextOffset = nextOffset; + if (offset >= fixedNextOffset) { + throw new IllegalArgumentException( + String.format( + "Attempting the read the kraft.version at an offset (%d) which kraft hasn't seen (%d)", + offset, + fixedNextOffset - 1 + ) + ); + } + + synchronized (kraftVersionHistory) { + return kraftVersionHistory.valueAtOrBefore(offset).orElse((short) 0); + } + } + + private void maybeLoadLog() { + while (log.endOffset().offset > nextOffset) { + LogFetchInfo info = log.read(nextOffset, Isolation.UNCOMMITTED); + try (RecordsIterator<?> iterator = new RecordsIterator<>( + info.records, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + while (iterator.hasNext()) { + Batch<?> batch = iterator.next(); + handleBatch(batch, OptionalLong.empty()); + nextOffset = batch.lastOffset() + 1; + } + } + } + } + + private void maybeLoadSnapshot() { + if ((nextOffset == 0 || nextOffset < log.startOffset()) && log.latestSnapshot().isPresent()) { + RawSnapshotReader rawSnapshot = log.latestSnapshot().get(); + // Clear the current state + synchronized (kraftVersionHistory) { + kraftVersionHistory.clear(); + } + synchronized (voterSetHistory) { + voterSetHistory.clear(); + } + + // Load the snapshot since the listener is at the start of the log or the log doesn't have the next entry. + try (SnapshotReader<?> reader = RecordsSnapshotReader.of( + rawSnapshot, + serde, + bufferSupplier, + maxBatchSizeBytes, + true // Validate batch CRC + ) + ) { + logger.info( + "Loading snapshot ({}) since log start offset ({}) is greater than the internal listener's next offset ({})", + reader.snapshotId(), + log.startOffset(), + nextOffset + ); + OptionalLong currentOffset = OptionalLong.of(reader.lastContainedLogOffset()); + while (reader.hasNext()) { + Batch<?> batch = reader.next(); + handleBatch(batch, currentOffset); + } + + nextOffset = reader.lastContainedLogOffset() + 1; + } + } + } + + private void handleBatch(Batch<?> batch, OptionalLong overrideOffset) { + int index = 0; Review Comment: I wonder if it is better to get the record offsets from the `Batch` object. This works because we know the partition is not compacted, but it still feels a bit suspicious. ########## raft/src/main/java/org/apache/kafka/raft/internals/History.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Objects; +import java.util.Optional; + +/** + * A object tracks values of {@code T} at different offsets. + */ +public interface History<T> { + /** + * Add a new value at a given offset. + * + * The provided {@code offset} must be greater than or equal to 0 and must be greater than the + * offset of all previous calls to this method. + * + * @param offset the offset + * @param value the value to store + * @throws IllegalArgumentException if the offset is not greater that all previous offsets + */ + public void addAt(long offset, T value); + + /** + * Returns the value that has the largest offset that is less than or equals to the provided + * offset. + * + * @param offset the offset + * @return the value if it exist, otherwise {@code Optional.empty()} + */ + public Optional<T> valueAtOrBefore(long offset); Review Comment: nit: we can drop `public` since it is an interface ########## raft/src/main/java/org/apache/kafka/raft/internals/VoterSetHistory.java: ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.internals; + +import java.util.Optional; + +/** + * A type for storing the historical value of the set of voters. + * + * This type can be use to keep track in-memory the sets for voters stored in the latest snapshot + * and log. This is useful when both generating a new snapshot at a given offset or when evaulating + * the latest set of voters. + */ +final public class VoterSetHistory implements History<VoterSet> { + private final Optional<VoterSet> staticVoterSet; + private final History<VoterSet> votersHistory = new TreeMapHistory<>(); + + VoterSetHistory(Optional<VoterSet> staticVoterSet) { + this.staticVoterSet = staticVoterSet; + } + + @Override + public void addAt(long offset, VoterSet voters) { + Optional<History.Entry<VoterSet>> lastEntry = votersHistory.lastEntry(); + if (lastEntry.isPresent() && lastEntry.get().offset() >= 0) { + // If the last voter set comes from the replicated log then the majorities must overlap. + // This ignores the static voter set and the bootstrapped voter set since they come from + // the configuration and the KRaft leader never guaranteed that they are the same across + // all replicas. + VoterSet lastVoterSet = lastEntry.get().value(); + if (!lastVoterSet.hasOverlappingMajority(voters)) { + throw new IllegalArgumentException( + String.format( + "Last voter set %s doesn't have an overlapping majority with the new voter set %s", + lastVoterSet, + voters + ) + ); + } + } + + votersHistory.addAt(offset, voters); + } + + /** + * Computes the value of the voter set at a given offset. + * + * This function will only return values provided through {@code addAt} and it would never + * include the {@code staticVoterSet} provided through the constructoer. + * + * @param offset the offset (inclusive) + * @return the voter set if one exist, otherwise {@code Optional.empty()} + */ + @Override + public Optional<VoterSet> valueAtOrBefore(long offset) { + return votersHistory.valueAtOrBefore(offset); + } + + @Override + public Optional<History.Entry<VoterSet>> lastEntry() { + Optional<History.Entry<VoterSet>> result = votersHistory.lastEntry(); + if (result.isPresent()) return result; + + return staticVoterSet.map(value -> new History.Entry<>(-1, value)); Review Comment: This seems a bit dangerous. Maybe we need a better type option. I think I had a comment about making `History` generic in the comparable type as well. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
