ss77892 commented on code in PR #11874:
URL: https://github.com/apache/ozone/pull/11874#discussion_r4514285509
##########
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java:
##########
@@ -753,63 +771,337 @@ protected boolean shouldBlockAndWaitAsyncReply(
return !HddsUtils.isReadOnly(request);
}
+ /**
+ * Sends the request to the given datanode for a caller that waits for the
+ * reply with {@link CompletableFuture#get()}: the response is delivered on
+ * the waiting thread, see {@link CallerRunsReplyFuture}.
+ */
@VisibleForTesting
public XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request, DatanodeDetails dn)
throws IOException, InterruptedException {
+ return sendCommandAsync(request, dn, new CallerRunsReplyFuture());
+ }
+
+ /**
+ * @param replyFuture completed with the reply; if it is an {@link Executor}
+ * the call's callbacks run on it, otherwise on gRPC's default
executor
+ */
+ private XceiverClientReply sendCommandAsync(
+ ContainerCommandRequestProto request, DatanodeDetails dn,
+ CompletableFuture<ContainerCommandResponseProto> replyFuture)
+ throws IOException, InterruptedException {
checkOpen(dn);
DatanodeID dnId = dn.getID();
if (LOG.isDebugEnabled()) {
LOG.debug("Send command {} to datanode {}",
request.getCmdType(), dn);
}
- final CompletableFuture<ContainerCommandResponseProto> replyFuture =
- new CompletableFuture<>();
semaphore.acquire();
long requestTime = Time.monotonicNow();
metrics.incrPendingContainerOpsMetrics(request.getCmdType());
+ final ChannelInfo channelInfo = dnChannelInfoMap.get(dnId);
+ final ReplyObserver reply = new ReplyObserver(request, dn, replyFuture,
requestTime);
+ if (replyFuture instanceof CallerRunsReplyFuture &&
!Span.current().getSpanContext().isValid()) {
+ // No trace context to send, so a pooled stream carries the same headers
as a new one would.
+ channelInfo.leaseStream().send(request, reply, (CallerRunsReplyFuture)
replyFuture,
+ System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout));
+ return new XceiverClientReply(replyFuture);
+ }
+
+ XceiverClientProtocolServiceStub stub =
channelInfo.getStub().withDeadlineAfter(timeout, TimeUnit.MILLISECONDS);
+ if (replyFuture instanceof Executor) {
+ stub = stub.withExecutor((Executor) replyFuture);
+ }
// create a new grpc message stream pair for each call.
- final StreamObserver<ContainerCommandRequestProto> requestObserver =
- dnChannelInfoMap.get(dnId).getStub().withDeadlineAfter(timeout,
TimeUnit.MILLISECONDS)
- .send(new StreamObserver<ContainerCommandResponseProto>() {
- @Override
- public void onNext(ContainerCommandResponseProto value) {
- replyFuture.complete(value);
- decreasePendingMetricsAndReleaseSemaphore();
- }
-
- @Override
- public void onError(Throwable t) {
- replyFuture.completeExceptionally(t);
- decreasePendingMetricsAndReleaseSemaphore();
- }
-
- @Override
- public void onCompleted() {
- if (!replyFuture.isDone()) {
- replyFuture.completeExceptionally(new IOException(
- "Stream completed but no reply for request " +
- processForDebug(request)));
- }
- }
-
- private void decreasePendingMetricsAndReleaseSemaphore() {
- metrics.decrPendingContainerOpsMetrics(request.getCmdType());
- long cost = Time.monotonicNow() - requestTime;
- metrics.addContainerOpsLatency(request.getCmdType(), cost);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Executed command {} on datanode {}, cost = {},
cmdType = {}",
- processForDebug(request), dn, cost,
request.getCmdType());
- }
- semaphore.release();
- }
- });
+ final StreamObserver<ContainerCommandRequestProto> requestObserver =
stub.send(reply);
requestObserver.onNext(request);
requestObserver.onCompleted();
return new XceiverClientReply(replyFuture);
}
+ /**
+ * Receives the reply to one request: completes its future, then releases
+ * its semaphore permit and updates the metrics.
+ */
+ @VisibleForTesting
+ final class ReplyObserver implements
StreamObserver<ContainerCommandResponseProto> {
+ private final ContainerCommandRequestProto request;
+ private final DatanodeDetails dn;
+ private final CompletableFuture<ContainerCommandResponseProto> replyFuture;
+ private final long requestTime;
+
+ ReplyObserver(ContainerCommandRequestProto request, DatanodeDetails dn,
+ CompletableFuture<ContainerCommandResponseProto> replyFuture, long
requestTime) {
+ this.request = request;
+ this.dn = dn;
+ this.replyFuture = replyFuture;
+ this.requestTime = requestTime;
+ }
+
+ @Override
+ public void onNext(ContainerCommandResponseProto value) {
+ replyFuture.complete(value);
+ decreasePendingMetricsAndReleaseSemaphore();
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ replyFuture.completeExceptionally(t);
+ decreasePendingMetricsAndReleaseSemaphore();
+ }
+
+ @Override
+ public void onCompleted() {
+ if (!replyFuture.isDone()) {
+ replyFuture.completeExceptionally(new IOException(
+ "Stream completed but no reply for request " +
+ processForDebug(request)));
+ }
+ }
+
+ private void decreasePendingMetricsAndReleaseSemaphore() {
+ metrics.decrPendingContainerOpsMetrics(request.getCmdType());
+ long cost = Time.monotonicNow() - requestTime;
+ metrics.addContainerOpsLatency(request.getCmdType(), cost);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Executed command {} on datanode {}, cost = {}, cmdType =
{}",
+ processForDebug(request), dn, cost, request.getCmdType());
+ }
+ semaphore.release();
+ }
+ }
+
+ /**
+ * A send stream kept open across requests, so a request does not pay for
+ * opening and closing an HTTP/2 stream. It carries one request at a time:
+ * the datanode serves the requests of a stream in order and replies to each
+ * one, so the next reply always belongs to the current request. The gRPC
+ * callbacks run on the current request's {@link CallerRunsReplyFuture}, as
+ * for a stream opened per request.
+ */
+ @VisibleForTesting
+ static final class PooledStream implements Executor,
+ ClientResponseObserver<ContainerCommandRequestProto,
ContainerCommandResponseProto> {
+ private final ChannelInfo owner;
+ private ClientCallStreamObserver<ContainerCommandRequestProto>
requestStream;
+ private final AtomicReference<ReplyObserver> current = new
AtomicReference<>();
+ private volatile CallerRunsReplyFuture currentFuture;
+ private volatile boolean closed;
+ private volatile long idleSinceNanos;
+
+ PooledStream(ChannelInfo owner) {
+ this.owner = owner;
+ }
+
+ @Override
+ public void
beforeStart(ClientCallStreamObserver<ContainerCommandRequestProto> stream) {
+ requestStream = stream;
+ }
+
+ void send(ContainerCommandRequestProto request, ReplyObserver reply,
CallerRunsReplyFuture future,
+ long deadlineNanos) {
+ future.setDeadline(deadlineNanos, this::cancel);
+ currentFuture = future;
+ current.set(reply);
+ if (closed && current.compareAndSet(reply, null)) {
+ // the stream ended after it was leased, before the request was
attached, so its callbacks did not fail it
+ currentFuture = null;
+ reply.onError(Status.UNAVAILABLE.withDescription("pooled stream closed
before the request was sent")
+ .asRuntimeException());
+ return;
+ }
+ requestStream.onNext(request);
+ }
+
+ @Override
+ public void execute(Runnable task) {
+ final CallerRunsReplyFuture future = current.get() != null ?
currentFuture : null;
+ if (future != null) {
+ future.execute(task);
+ } else {
+ task.run();
+ }
+ }
+
+ @Override
+ public void onNext(ContainerCommandResponseProto value) {
+ final ReplyObserver reply = current.getAndSet(null);
+ currentFuture = null;
+ // back in the pool before the permit is released, so the thread woken
by the permit finds the stream
+ owner.releaseStream(this);
+ reply.onNext(value);
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ closed = true;
+ final ReplyObserver reply = current.getAndSet(null);
+ currentFuture = null;
+ if (reply != null) {
+ reply.onError(t);
+ }
+ }
+
+ @Override
+ public void onCompleted() {
+ closed = true;
+ final ReplyObserver reply = current.getAndSet(null);
+ currentFuture = null;
+ if (reply != null) {
+ reply.onCompleted();
+ }
+ }
+
+ boolean isClosed() {
+ return closed;
+ }
+
+ void setIdleSince(long nanos) {
+ idleSinceNanos = nanos;
+ }
+
+ long getIdleSince() {
+ return idleSinceNanos;
+ }
+
+ /** Ends an idle stream, the datanode then completes it. */
+ void halfClose() {
+ closed = true;
+ requestStream.onCompleted();
+ }
+
+ /** Called when the request timed out or its waiter gave up; the stream
fails the request and is dropped. */
+ private void cancel() {
+ closed = true;
+ requestStream.cancel("request abandoned by the client", null);
Review Comment:
Fixed, thanks. The abandon path now takes the request's `ReplyObserver` and
only acts if `current.compareAndSet(reply, null)` succeeds, so it can't cancel
a stream that its reply has already returned to the pool. The winner also fails
the request and releases the permit itself.
`abandonedRequestDoesNotCancelTheNextRequestOnTheStream` covers it and fails
without the fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]