hachikuji commented on a change in pull request #11688: URL: https://github.com/apache/kafka/pull/11688#discussion_r803159171
########## File path: clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java ########## @@ -699,11 +702,15 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture<ByteBuffer> fut return sendSyncGroupRequest(requestBuilder); } - private RequestFuture<ByteBuffer> onJoinLeader(JoinGroupResponse joinResponse) { + private RequestFuture<ByteBuffer> onLeaderElected(JoinGroupResponse joinResponse) { try { // perform the leader synchronization and send back the assignment for the group - Map<String, ByteBuffer> groupAssignment = performAssignment(joinResponse.data().leader(), joinResponse.data().protocolName(), - joinResponse.data().members()); + Map<String, ByteBuffer> groupAssignment = onLeaderElected( + joinResponse.data().leader(), + joinResponse.data().protocolName(), + joinResponse.data().members(), + joinResponse.data().skipAssignment() + ); Review comment: Do you think it is worthwhile validating that `groupAssignment` is empty when `skipAssignment` is set? ########## File path: clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java ########## @@ -1691,6 +1717,64 @@ public void testMetadataChangeTriggersRebalance() { assertTrue(coordinator.rejoinNeededOrPending()); } + @Test + public void testStaticLeaderRejoinsGroupAndCanTriggersRebalance() { + // ensure metadata is up-to-date for leader + subscriptions.subscribe(singleton(topic1), rebalanceListener); + client.updateMetadata(metadataResponse); + + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); + + // the leader is responsible for picking up metadata changes and forcing a group rebalance. + // note that `MockPartitionAssignor.prepare` is not called therefore calling `MockPartitionAssignor.assign` + // will throw a IllegalStateException. this indirectly verifies that `assign` is correctly skipped. + Map<String, List<String>> memberSubscriptions = singletonMap(consumerId, singletonList(topic1)); + client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, true, Errors.NONE)); + client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + + coordinator.poll(time.timer(Long.MAX_VALUE)); + + assertFalse(coordinator.rejoinNeededOrPending()); + + // a new partition is added to the topic + metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWith(1, singletonMap(topic1, 2)), false, time.milliseconds()); + coordinator.maybeUpdateSubscriptionMetadata(); + + // we should detect the change and ask for reassignment + assertTrue(coordinator.rejoinNeededOrPending()); + } + + @Test + public void testStaticLeaderRejoinsGroupAndCanDetectMetadataChangesForOtherMembers() { + // ensure metadata is up-to-date for leader + subscriptions.subscribe(singleton(topic1), rebalanceListener); + client.updateMetadata(metadataResponse); + + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); + + // the leader is responsible for picking up metadata changes and forcing a group rebalance. + // note that `MockPartitionAssignor.prepare` is not called therefore calling `MockPartitionAssignor.assign` + // will throw a IllegalStateException. this indirectly verifies that `assign` is correctly skipped. + Map<String, List<String>> memberSubscriptions = new HashMap<>(); + memberSubscriptions.put(consumerId, singletonList(topic1)); + memberSubscriptions.put(consumerId2, singletonList(topic2)); + client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, true, Errors.NONE)); + client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + + coordinator.poll(time.timer(Long.MAX_VALUE)); + + assertFalse(coordinator.rejoinNeededOrPending()); + Review comment: Could we add an assertion for `SubscriptionState.metadataTopics`? ########## File path: clients/src/main/resources/common/message/JoinGroupResponse.json ########## @@ -49,6 +51,8 @@ "about": "The group protocol selected by the coordinator." }, { "name": "Leader", "type": "string", "versions": "0+", "about": "The leader of the group." }, + { "name": "SkipAssignment", "type": "bool", "versions": "9+", "default": "false", + "about": "True is the leader must skip running the assignment." }, Review comment: nit: is -> if ########## File path: core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala ########## @@ -1791,4 +1793,34 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertTrue(records2.count() == 1 && records2.records(tp).asScala.head.offset == 1, "Expected consumer2 to consume one message from offset 1, which is the committed offset of consumer1") } + + @Test + def testStaticConsumerDetectsNewPartitionCreatedAfterRestart(): Unit = { + val foo = "foo" + val foo0 = new TopicPartition(foo, 0) + val foo1 = new TopicPartition(foo, 1) + + val admin = createAdminClient() + admin.createTopics(Seq(new NewTopic(foo, 1, 1.asInstanceOf[Short])).asJava).all.get Review comment: nit: I think 1.toShort works? ########## File path: clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java ########## @@ -645,6 +651,15 @@ private void maybeUpdateGroupSubscription(String assignorName, isLeader = true; + if (skipAssignment) { + log.info("Skipped assignment for returning static leader at generation {}. The static leader " + + "will continue with its existing assignment.", generation().generationId); + assignmentSnapshot = metadataSnapshot; + return Collections.emptyMap(); + } + + Map<String, ByteBuffer> groupAssignment = new HashMap<>(); Review comment: nit: seemed reasonable at its original location -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org