chia7712 commented on code in PR #16127:
URL: https://github.com/apache/kafka/pull/16127#discussion_r1666635563
##########
core/src/test/java/kafka/testkit/KafkaClusterTestKit.java:
##########
@@ -390,7 +392,7 @@ public void format() throws Exception {
BrokerServer broker = entry.getValue();
futures.add(executorService.submit(() -> {
formatNode(broker.sharedServer().metaPropsEnsemble(),
- !nodes().brokerNodes().get(entry.getKey()).combined());
+
!nodes().brokerNodes().get(entry.getKey()).combined());
Review Comment:
ditto
##########
tools/src/test/java/org/apache/kafka/tools/TopicCommandIntegrationTest.java:
##########
@@ -86,652 +85,952 @@
@Tag("integration")
@SuppressWarnings("deprecation") // Added for Scala 2.12 compatibility for
usages of JavaConverters
-public class TopicCommandIntegrationTest extends
kafka.integration.KafkaServerTestHarness implements Logging, RackAwareTest {
+@ExtendWith(ClusterTestExtensions.class)
+public class TopicCommandIntegrationTest {
private final short defaultReplicationFactor = 1;
private final int defaultNumPartitions = 1;
- private TopicCommand.TopicService topicService;
- private Admin adminClient;
- private String bootstrapServer;
- private String testTopicName;
- private Buffer<KafkaBroker> scalaBrokers;
- private Seq<ControllerServer> scalaControllers;
- /**
- * Implementations must override this method to return a set of
KafkaConfigs. This method will be invoked for every
- * test and should not reuse previous configurations unless they select
their ports randomly when servers are started.
- *
- * Note the replica fetch max bytes is set to `1` in order to throttle the
rate of replication for test
- * `testDescribeUnderReplicatedPartitionsWhenReassignmentIsInProgress`.
- */
- @Override
- public scala.collection.Seq<KafkaConfig> generateConfigs() {
- Map<Integer, String> rackInfo = new HashMap<>();
- rackInfo.put(0, "rack1");
- rackInfo.put(1, "rack2");
- rackInfo.put(2, "rack2");
- rackInfo.put(3, "rack1");
- rackInfo.put(4, "rack3");
- rackInfo.put(5, "rack3");
-
- List<Properties> brokerConfigs = ToolsTestUtils
- .createBrokerProperties(6, zkConnectOrNull(), rackInfo,
defaultNumPartitions, defaultReplicationFactor);
-
- List<KafkaConfig> configs = new ArrayList<>();
- for (Properties props : brokerConfigs) {
- props.put(REPLICA_FETCH_MAX_BYTES_CONFIG, "1");
- configs.add(KafkaConfig.fromProps(props));
- }
- return JavaConverters.asScalaBuffer(configs).toSeq();
- }
+ private final ClusterInstance clusterInstance;
private TopicCommand.TopicCommandOptions
buildTopicCommandOptionsWithBootstrap(String... opts) {
+ String bootstrapServer = clusterInstance.bootstrapServers();
String[] finalOptions = Stream.concat(Arrays.stream(opts),
Stream.of("--bootstrap-server", bootstrapServer)
).toArray(String[]::new);
return new TopicCommand.TopicCommandOptions(finalOptions);
}
- @BeforeEach
- public void setUp(TestInfo info) {
- super.setUp(info);
- // create adminClient
- Properties props = new Properties();
- bootstrapServer = bootstrapServers(listenerName());
- props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
bootstrapServer);
- adminClient = Admin.create(props);
- topicService = new TopicCommand.TopicService(props,
Optional.of(bootstrapServer));
- testTopicName = String.format("%s-%s",
info.getTestMethod().get().getName(),
org.apache.kafka.test.TestUtils.randomString(10));
- scalaBrokers = brokers();
- scalaControllers = controllerServers();
+ static List<ClusterConfig> generate1() {
+ Map<String, String> serverProp = new HashMap<>();
+ serverProp.put(REPLICA_FETCH_MAX_BYTES_CONFIG, "1"); // if config name
error, no exception throw
+
+ Map<Integer, Map<String, String>> rackInfo = new HashMap<>();
+ Map<String, String> infoPerBroker1 = new HashMap<>();
+ infoPerBroker1.put("broker.rack", "rack1");
+ Map<String, String> infoPerBroker2 = new HashMap<>();
+ infoPerBroker2.put("broker.rack", "rack2");
+ Map<String, String> infoPerBroker3 = new HashMap<>();
+ infoPerBroker3.put("broker.rack", "rack2");
+ Map<String, String> infoPerBroker4 = new HashMap<>();
+ infoPerBroker4.put("broker.rack", "rack1");
+ Map<String, String> infoPerBroker5 = new HashMap<>();
+ infoPerBroker5.put("broker.rack", "rack3");
+ Map<String, String> infoPerBroker6 = new HashMap<>();
+ infoPerBroker6.put("broker.rack", "rack3");
+
+ rackInfo.put(0, infoPerBroker1);
+ rackInfo.put(1, infoPerBroker2);
+ rackInfo.put(2, infoPerBroker3);
+ rackInfo.put(3, infoPerBroker4);
+ rackInfo.put(4, infoPerBroker5);
+ rackInfo.put(5, infoPerBroker6);
+
+ return Collections.singletonList(ClusterConfig.defaultBuilder()
+ .setBrokers(6)
+ .setServerProperties(serverProp)
+ .setPerServerProperties(rackInfo)
+ .build()
+ );
}
- @AfterEach
- public void close() throws Exception {
- if (topicService != null)
- topicService.close();
- if (adminClient != null)
- adminClient.close();
+ TopicCommandIntegrationTest(ClusterInstance clusterInstance) {
+ this.clusterInstance = clusterInstance;
}
- @ParameterizedTest
- @ValueSource(strings = {"zk", "kraft"})
- public void testCreate(String quorum) throws Exception {
- TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, 2, 1,
- scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
- );
-
assertTrue(adminClient.listTopics().names().get().contains(testTopicName),
- "Admin client didn't see the created topic. It saw: " +
adminClient.listTopics().names().get());
- }
+ @ClusterTemplate("generate1")
+ public void testCreate() throws InterruptedException, ExecutionException {
+ String testTopicName =
org.apache.kafka.test.TestUtils.getCurrentFunctionName() + "-" +
+ org.apache.kafka.test.TestUtils.randomString(10);
- @ParameterizedTest
- @ValueSource(strings = {"zk", "kraft"})
- public void testCreateWithDefaults(String quorum) throws Exception {
- TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, defaultNumPartitions, defaultReplicationFactor,
- scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
- );
- List<TopicPartitionInfo> partitions = adminClient
- .describeTopics(Collections.singletonList(testTopicName))
- .allTopicNames()
- .get()
- .get(testTopicName)
- .partitions();
- assertEquals(defaultNumPartitions, partitions.size(), "Unequal
partition size: " + partitions.size());
- assertEquals(defaultReplicationFactor, (short)
partitions.get(0).replicas().size(), "Unequal replication factor: " +
partitions.get(0).replicas().size());
- }
+ try (Admin adminClient = clusterInstance.createAdminClient()) {
+ adminClient.createTopics(Collections.singletonList(new
NewTopic(testTopicName, defaultNumPartitions, defaultReplicationFactor)));
- @ParameterizedTest
- @ValueSource(strings = {"zk", "kraft"})
- public void testCreateWithDefaultReplication(String quorum) throws
Exception {
- TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, 2, defaultReplicationFactor,
- scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
- );
- List<TopicPartitionInfo> partitions = adminClient
- .describeTopics(Collections.singletonList(testTopicName))
- .allTopicNames()
- .get()
- .get(testTopicName)
- .partitions();
- assertEquals(2, partitions.size(), "Unequal partition size: " +
partitions.size());
- assertEquals(defaultReplicationFactor, (short)
partitions.get(0).replicas().size(), "Unequal replication factor: " +
partitions.get(0).replicas().size());
+ clusterInstance.waitForTopic(testTopicName, defaultNumPartitions);
+
Assertions.assertTrue(adminClient.listTopics().names().get().contains(testTopicName),
+ "Admin client didn't see the created topic. It saw: " +
adminClient.listTopics().names().get());
+
+ adminClient.deleteTopics(Collections.singletonList(testTopicName));
+ clusterInstance.waitForTopic(testTopicName, 0);
+
Assertions.assertTrue(adminClient.listTopics().names().get().isEmpty(),
+ "Admin client see the created topic. It saw: " +
adminClient.listTopics().names().get());
+ }
}
- @ParameterizedTest
- @ValueSource(strings = {"zk", "kraft"})
- public void testCreateWithDefaultPartitions(String quorum) throws
Exception {
- TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, defaultNumPartitions, 2,
- scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
- );
- List<TopicPartitionInfo> partitions = adminClient
- .describeTopics(Collections.singletonList(testTopicName))
- .allTopicNames()
- .get()
- .get(testTopicName)
- .partitions();
-
- assertEquals(defaultNumPartitions, partitions.size(), "Unequal
partition size: " + partitions.size());
- assertEquals(2, (short) partitions.get(0).replicas().size(),
"Partitions not replicated: " + partitions.get(0).replicas().size());
+ @ClusterTemplate("generate1")
+ public void testCreateWithDefaults() throws InterruptedException,
ExecutionException {
+ String testTopicName =
org.apache.kafka.test.TestUtils.getCurrentFunctionName() + "-" +
+ org.apache.kafka.test.TestUtils.randomString(10);
+
+ try (Admin adminClient = clusterInstance.createAdminClient()) {
+ adminClient.createTopics(Collections.singletonList(new
NewTopic(testTopicName, defaultNumPartitions, defaultReplicationFactor)));
+
+ clusterInstance.waitForTopic(testTopicName, defaultNumPartitions);
+
Assertions.assertTrue(adminClient.listTopics().names().get().contains(testTopicName),
+ "Admin client didn't see the created topic. It saw: " +
adminClient.listTopics().names().get());
+
+ List<TopicPartitionInfo> partitions = adminClient
+ .describeTopics(Collections.singletonList(testTopicName))
+ .allTopicNames()
+ .get()
+ .get(testTopicName)
+ .partitions();
+ Assertions.assertEquals(defaultNumPartitions, partitions.size(),
"Unequal partition size: " + partitions.size());
+ Assertions.assertEquals(defaultReplicationFactor, (short)
partitions.get(0).replicas().size(), "Unequal replication factor: " +
partitions.get(0).replicas().size());
+
+ adminClient.deleteTopics(Collections.singletonList(testTopicName));
+ clusterInstance.waitForTopic(testTopicName, 0);
+
Assertions.assertTrue(adminClient.listTopics().names().get().isEmpty(),
+ "Admin client see the created topic. It saw: " +
adminClient.listTopics().names().get());
+ }
}
- @ParameterizedTest
- @ValueSource(strings = {"zk", "kraft"})
- public void testCreateWithConfigs(String quorum) throws Exception {
- ConfigResource configResource = new
ConfigResource(ConfigResource.Type.TOPIC, testTopicName);
- Properties topicConfig = new Properties();
- topicConfig.put(TopicConfig.DELETE_RETENTION_MS_CONFIG, "1000");
+ @ClusterTemplate("generate1")
+ public void testCreateWithDefaultReplication() throws
InterruptedException, ExecutionException {
+ String testTopicName =
org.apache.kafka.test.TestUtils.getCurrentFunctionName() + "-" +
+ org.apache.kafka.test.TestUtils.randomString(10);
+
+ try (Admin adminClient = clusterInstance.createAdminClient()) {
+ Buffer<KafkaBroker> scalaBrokers =
JavaConverters.asScalaIteratorConverter(
+ clusterInstance.brokers().values().iterator())
+ .asScala().toBuffer();
+ Seq<ControllerServer> scalaControllers =
JavaConverters.asScalaIteratorConverter(
+ clusterInstance.controllers().values().iterator())
+ .asScala().toSeq();
+
+ TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, 2, defaultReplicationFactor,
+ scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
+ );
+ List<TopicPartitionInfo> partitions = adminClient
+ .describeTopics(Collections.singletonList(testTopicName))
+ .allTopicNames()
+ .get()
+ .get(testTopicName)
+ .partitions();
+ assertEquals(2, partitions.size(), "Unequal partition size: " +
partitions.size());
+ assertEquals(defaultReplicationFactor, (short)
partitions.get(0).replicas().size(), "Unequal replication factor: " +
partitions.get(0).replicas().size());
+ }
+ }
- TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, 2, 2,
- scala.collection.immutable.Map$.MODULE$.empty(), topicConfig
- );
+ @ClusterTemplate("generate1")
+ public void testCreateWithDefaultPartitions() throws InterruptedException,
ExecutionException {
+ String testTopicName =
org.apache.kafka.test.TestUtils.getCurrentFunctionName() + "-" +
+ org.apache.kafka.test.TestUtils.randomString(10);
+
+ try (Admin adminClient = clusterInstance.createAdminClient()) {
+ Buffer<KafkaBroker> scalaBrokers =
JavaConverters.asScalaIteratorConverter(
+ clusterInstance.brokers().values().iterator())
+ .asScala().toBuffer();
+ Seq<ControllerServer> scalaControllers =
JavaConverters.asScalaIteratorConverter(
+ clusterInstance.controllers().values().iterator())
+ .asScala().toSeq();
+ TestUtils.createTopicWithAdmin(adminClient, testTopicName,
scalaBrokers, scalaControllers, defaultNumPartitions, 2,
+ scala.collection.immutable.Map$.MODULE$.empty(), new
Properties()
+ );
+ List<TopicPartitionInfo> partitions = adminClient
+ .describeTopics(Collections.singletonList(testTopicName))
+ .allTopicNames()
+ .get()
+ .get(testTopicName)
+ .partitions();
+
+ assertEquals(defaultNumPartitions, partitions.size(), "Unequal
partition size: " + partitions.size());
+ assertEquals(2, (short) partitions.get(0).replicas().size(),
"Partitions not replicated: " + partitions.get(0).replicas().size());
+ }
+ }
- Config configs =
adminClient.describeConfigs(Collections.singleton(configResource)).all().get().get(configResource);
- assertEquals(1000,
Integer.valueOf(configs.get("delete.retention.ms").value()),
- "Config not set correctly: " +
configs.get("delete.retention.ms").value());
+ @ClusterTemplate("generate1")
+ public void testCreateWithConfigs() throws Exception {
+ String testTopicName =
org.apache.kafka.test.TestUtils.getCurrentFunctionName() + "-" +
Review Comment:
It seems to me `randomString` is good enough. Or we can add a static count
to generate random topic name
##########
core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java:
##########
@@ -189,7 +189,9 @@ public void start() {
if (started.compareAndSet(false, true)) {
clusterTestKit.startup();
kafka.utils.TestUtils.waitUntilTrue(
- () ->
this.clusterTestKit.brokers().get(0).brokerState() == BrokerState.RUNNING,
Review Comment:
nice finding. Maybe you can file a MINOR patch for it, since this PR needs
more time to be completed
##########
core/src/test/java/kafka/testkit/KafkaClusterTestKit.java:
##########
@@ -356,18 +357,19 @@ private static void setupNodeDirectories(File
baseDirectory,
private final TestKitNodes nodes;
private final Map<Integer, ControllerServer> controllers;
private final Map<Integer, BrokerServer> brokers;
+
private final ControllerQuorumVotersFutureManager
controllerQuorumVotersFutureManager;
private final File baseDirectory;
private final SimpleFaultHandlerFactory faultHandlerFactory;
private KafkaClusterTestKit(
- ExecutorService executorService,
- TestKitNodes nodes,
- Map<Integer, ControllerServer> controllers,
- Map<Integer, BrokerServer> brokers,
- ControllerQuorumVotersFutureManager
controllerQuorumVotersFutureManager,
- File baseDirectory,
- SimpleFaultHandlerFactory faultHandlerFactory
+ ExecutorService executorService,
Review Comment:
please revert those changes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
