dajac commented on code in PR #14364: URL: https://github.com/apache/kafka/pull/14364#discussion_r1342667683
########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); Review Comment: Would it make sense to just call `membershipManager.transitionToFailed();` in all errors instead of having this one? At the movement, the handling is a little inconsistent. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); Review Comment: This message is not consistent with the others. Should it also start with `GroupHeartbeatRequest failed due to...`? I would also replace `Retrying` by something like `Will attempt to find the coordinator again and retry`. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: + // retry + message = String.format("Heartbeat was not successful because the coordinator node %s is " + + "loading." + + "Retrying", coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + break; + case GROUP_AUTHORIZATION_FAILED: + GroupAuthorizationException exception = + GroupAuthorizationException.forGroupId(membershipManager.groupId()); + logger.error("GroupHeartbeatRequest failed due to group authorization failure: {}", exception.getMessage()); + nonRetriableErrorHandler.handle(exception, errorMessage); + break; + case INVALID_REQUEST: + logger.error("GroupHeartbeatRequest failed due to invalid request error: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.INVALID_REQUEST.exception(), errorMessage); + break; + case GROUP_MAX_SIZE_REACHED: + logger.error("GroupHeartbeatRequest failed due to the max group size limit: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.GROUP_MAX_SIZE_REACHED.exception(), errorMessage); + break; + case UNSUPPORTED_ASSIGNOR: + logger.error("GroupHeartbeatRequest failed due to unsupported assignor {}: {}", + membershipManager.assignorSelection(), errorMessage); + nonRetriableErrorHandler.handle(Errors.UNSUPPORTED_ASSIGNOR.exception(), errorMessage); + break; + case UNRELEASED_INSTANCE_ID: + logger.error("GroupHeartbeatRequest failed due to the instance id {} was not released: {}", + membershipManager.groupInstanceId().orElse("null"), errorMessage); + nonRetriableErrorHandler.handle(Errors.UNRELEASED_INSTANCE_ID.exception(), errorMessage); + break; + case FENCED_MEMBER_EPOCH: + case UNKNOWN_MEMBER_ID: + membershipManager.transitionToFenced(); Review Comment: Should we also log something here? ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: + // retry + message = String.format("Heartbeat was not successful because the coordinator node %s is " + + "loading." + + "Retrying", coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + break; + case GROUP_AUTHORIZATION_FAILED: + GroupAuthorizationException exception = + GroupAuthorizationException.forGroupId(membershipManager.groupId()); + logger.error("GroupHeartbeatRequest failed due to group authorization failure: {}", exception.getMessage()); + nonRetriableErrorHandler.handle(exception, errorMessage); + break; + case INVALID_REQUEST: + logger.error("GroupHeartbeatRequest failed due to invalid request error: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.INVALID_REQUEST.exception(), errorMessage); Review Comment: I don't really understand how the error message is handled here. Is it going to be added as the message of the exception later on? When I mentioned this in my earlier comment, I means doing this `Errors.INVALID_REQUEST.exception(errorMessage)`. This uses the provided error message. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager only emits heartbeat when the member is in a group, tries to join or rejoin a group. + * If the member does not have groupId configured, got kicked out of the group, or encountering fatal exceptions, the + * heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>If the member completes the assignment changes, i.e. revocation and assignment, a heartbeat request will be + * sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final Set<Errors> fatalErrors = new HashSet<>(Arrays.asList( + Errors.GROUP_AUTHORIZATION_FAILED, + Errors.INVALID_REQUEST, + Errors.GROUP_MAX_SIZE_REACHED, + Errors.UNSUPPORTED_ASSIGNOR, + Errors.UNRELEASED_INSTANCE_ID)); + + private final int rebalanceTimeoutMs; Review Comment: Does the 👍🏻 mean that you will add it? ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } Review Comment: My understanding is that we basically retry on all exceptions here. Am I correct? It seems to me that we could also get non-retriable exceptions here (e.g. UnsupportedVersionException, etc.). How do we handle those? ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: Review Comment: When we get this one, I understand that we will mark the coordinator as unknown to rediscover it. Is it going to apply the exponential backoff after that? ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: + // retry + message = String.format("Heartbeat was not successful because the coordinator node %s is " + + "loading." + + "Retrying", coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + break; + case GROUP_AUTHORIZATION_FAILED: + GroupAuthorizationException exception = + GroupAuthorizationException.forGroupId(membershipManager.groupId()); + logger.error("GroupHeartbeatRequest failed due to group authorization failure: {}", exception.getMessage()); + nonRetriableErrorHandler.handle(exception, errorMessage); + break; + case INVALID_REQUEST: + logger.error("GroupHeartbeatRequest failed due to invalid request error: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.INVALID_REQUEST.exception(), errorMessage); + break; + case GROUP_MAX_SIZE_REACHED: + logger.error("GroupHeartbeatRequest failed due to the max group size limit: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.GROUP_MAX_SIZE_REACHED.exception(), errorMessage); + break; + case UNSUPPORTED_ASSIGNOR: + logger.error("GroupHeartbeatRequest failed due to unsupported assignor {}: {}", + membershipManager.assignorSelection(), errorMessage); + nonRetriableErrorHandler.handle(Errors.UNSUPPORTED_ASSIGNOR.exception(), errorMessage); + break; + case UNRELEASED_INSTANCE_ID: + logger.error("GroupHeartbeatRequest failed due to the instance id {} was not released: {}", + membershipManager.groupInstanceId().orElse("null"), errorMessage); + nonRetriableErrorHandler.handle(Errors.UNRELEASED_INSTANCE_ID.exception(), errorMessage); + break; + case FENCED_MEMBER_EPOCH: + case UNKNOWN_MEMBER_ID: + membershipManager.transitionToFenced(); + default: Review Comment: We also need to handle `UnsupportedVersionException` error, I think. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: + // retry + message = String.format("Heartbeat was not successful because the coordinator node %s is " + + "loading." + + "Retrying", coordinatorRequestManager.coordinator()); Review Comment: nit: This one is also inconsistent. `+ "Retrying"` could be merged with the previous string. A space misses between `loading.` and `Retrying`. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: + // retry + message = String.format("Heartbeat was not successful because the coordinator node %s is " + + "loading." + + "Retrying", coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + break; + case GROUP_AUTHORIZATION_FAILED: + GroupAuthorizationException exception = + GroupAuthorizationException.forGroupId(membershipManager.groupId()); + logger.error("GroupHeartbeatRequest failed due to group authorization failure: {}", exception.getMessage()); + nonRetriableErrorHandler.handle(exception, errorMessage); + break; + case INVALID_REQUEST: + logger.error("GroupHeartbeatRequest failed due to invalid request error: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.INVALID_REQUEST.exception(), errorMessage); + break; + case GROUP_MAX_SIZE_REACHED: + logger.error("GroupHeartbeatRequest failed due to the max group size limit: {}", errorMessage); + nonRetriableErrorHandler.handle(Errors.GROUP_MAX_SIZE_REACHED.exception(), errorMessage); + break; + case UNSUPPORTED_ASSIGNOR: + logger.error("GroupHeartbeatRequest failed due to unsupported assignor {}: {}", + membershipManager.assignorSelection(), errorMessage); + nonRetriableErrorHandler.handle(Errors.UNSUPPORTED_ASSIGNOR.exception(), errorMessage); + break; Review Comment: I also wonder if having a tailored error message for each error is needed. An alternative would be to group errors and have a generic message for the fatal ones for instance. What do you think? ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, + * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * group, is trying to join a group, or is in the process of reconciling the assignment changes.</p> + * + * <p>If the member got kick out of a group, it will try to give up the current assignment by invoking {@code + * OnPartitionsLost} because reattempting to join again with a zero epoch.</p> + * + * <p>If the member does not have groupId configured or encountering fatal exceptions, a heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>When the member completes the assignment reconciliation, the {@link HeartbeatRequestState} will be reset so + * that a heartbeat will be sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final int rebalanceTimeoutMs; + + private final CoordinatorRequestManager coordinatorRequestManager; + private final SubscriptionState subscriptions; + private final HeartbeatRequestState heartbeatRequestState; + private final MembershipManager membershipManager; + private final ErrorEventHandler nonRetriableErrorHandler; + + public HeartbeatRequestManager( + final Time time, + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final ErrorEventHandler nonRetriableErrorHandler) { + this.coordinatorRequestManager = coordinatorRequestManager; + this.logger = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, + retryBackoffMaxMs, rebalanceTimeoutMs); + } + + // Visible for testing + HeartbeatRequestManager( + final LogContext logContext, + final ConsumerConfig config, + final CoordinatorRequestManager coordinatorRequestManager, + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final HeartbeatRequestState heartbeatRequestState, + final ErrorEventHandler nonRetriableErrorHandler) { + this.logger = logContext.logger(this.getClass()); + this.subscriptions = subscriptions; + this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); + this.coordinatorRequestManager = coordinatorRequestManager; + this.heartbeatRequestState = heartbeatRequestState; + this.membershipManager = membershipManager; + this.nonRetriableErrorHandler = nonRetriableErrorHandler; + } + + /** + * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat + * request. + * <ol> + * <li>If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.</li> + * <li>If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.</li> + * <li>If the member's heartbeat timer has not expired, It will return the remaining time left on the + * heartbeat timer.</li> + * <li>If the member can send a heartbeat, the timer is set to the current heartbeat interval.</li> + * </ol> + */ + @Override + public NetworkClientDelegate.PollResult poll(long currentTimeMs) { + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { + return new NetworkClientDelegate.PollResult( + Long.MAX_VALUE, Collections.emptyList()); + } + + // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be + // implemented either with or after the partition reconciliation logic. + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { + return new NetworkClientDelegate.PollResult( + heartbeatRequestState.nextHeartbeatMs(currentTimeMs), + Collections.emptyList()); + } + this.heartbeatRequestState.onSendAttempt(currentTimeMs); + NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); + return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); + } + + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { + // TODO: We only need to send this field once unless the first request failed. + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() + .setGroupId(membershipManager.groupId()) + .setMemberEpoch(membershipManager.memberEpoch()) + .setMemberId(membershipManager.memberId()) + .setRebalanceTimeoutMs(rebalanceTimeoutMs); + + membershipManager.groupInstanceId().ifPresent(data::setInstanceId); + + if (this.subscriptions.hasPatternSubscription()) { + // We haven't discsussed how Regex is stored in the consumer. We could do it in the subscriptionState + // , in the memberStateManager, or here. + // data.setSubscribedTopicRegex(regex) + } else { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + } + + this.membershipManager.assignorSelection().serverAssignor().ifPresent(data::setServerAssignor); + + NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( + new ConsumerGroupHeartbeatRequest.Builder(data), + coordinatorRequestManager.coordinator()); + + request.future().whenComplete((response, exception) -> { + if (exception == null) { + onResponse((ConsumerGroupHeartbeatResponse) response.responseBody(), response.receivedTimeMs()); + } else { + onFailure(exception, response.receivedTimeMs()); + } + }); + return request; + } + + private void onFailure(final Throwable exception, final long responseTimeMs) { + this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + logger.warn("Failed to send heartbeat to coordinator node {} due to error: {}", + coordinatorRequestManager.coordinator(), exception.getMessage()); + } + + private void onResponse(final ConsumerGroupHeartbeatResponse response, long currentTimeMs) { + if (Errors.forCode(response.data().errorCode()) == Errors.NONE) { + this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); + this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); + this.heartbeatRequestState.resetTimer(); + this.membershipManager.updateState(response.data()); + return; + } + onErrorResponse(response, currentTimeMs); + } + + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, + final long currentTimeMs) { + this.heartbeatRequestState.onFailedAttempt(currentTimeMs); + Errors error = Errors.forCode(response.data().errorCode()); + maybeTransitionToFailureState(error); + String errorMessage = response.data().errorMessage(); + switch(error) { + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + String message = String.format("Coordinator node %s is either not started or not valid. Retrying", + coordinatorRequestManager.coordinator()); + logInfo(errorMessage, response, currentTimeMs); + coordinatorRequestManager.markCoordinatorUnknown(response.data().errorMessage(), currentTimeMs); + break; + case COORDINATOR_LOAD_IN_PROGRESS: Review Comment: nit: We usually put an empty line between cases. It makes it a bit more readable. ########## clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java: ########## @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; +import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * <p>Manages the request creation and response handling for the heartbeat. The module creates a + * {@link ConsumerGroupHeartbeatRequest} using the state stored in the {@link MembershipManager} and enqueue it to + * the network queue to be sent out. Once the response is received, the module will update the state in the + * {@link MembershipManager} and handle any errors.</p> + * + * <p>The manager only emits heartbeat when the member is in a group, tries to join or rejoin a group. + * If the member does not have groupId configured, got kicked out of the group, or encountering fatal exceptions, the + * heartbeat will not be sent.</p> + * + * <p>If the coordinator not is not found, we will skip sending the heartbeat and try to find a coordinator first.</p> + * + * <p>If the heartbeat failed due to retriable errors, such as, TimeoutException. The subsequent attempt will be + * backoff exponentially.</p> + * + * <p>If the member completes the assignment changes, i.e. revocation and assignment, a heartbeat request will be + * sent in the next event loop.</p> + * + * <p>See {@link HeartbeatRequestState} for more details.</p> + */ +public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; + private final Set<Errors> fatalErrors = new HashSet<>(Arrays.asList( + Errors.GROUP_AUTHORIZATION_FAILED, + Errors.INVALID_REQUEST, + Errors.GROUP_MAX_SIZE_REACHED, + Errors.UNSUPPORTED_ASSIGNOR, + Errors.UNRELEASED_INSTANCE_ID)); + + private final int rebalanceTimeoutMs; Review Comment: Does the 👍🏻 mean that you will add it? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
