This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 500a64aa997 [fix](arrow-flight) Report the real client address of an
Arrow Flight SQL session (#67576)
500a64aa997 is described below
commit 500a64aa99756dab57e31304c0ac02e8a5c527ef
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Sep 10 15:39:55 2026 +0800
[fix](arrow-flight) Report the real client address of an Arrow Flight SQL
session (#67576)
### What problem does this PR solve?
Problem Summary:
`FlightSqlChannel` had `getRemoteIp()` and `getRemoteHostPortString()`
stubbed out as `0.0.0.0` and
`0.0.0.0:0`, and `FlightSqlConnectContext` routed every client-address
accessor through them. So an
Arrow Flight SQL session reported `0.0.0.0:0` everywhere an operator
looks for one:
- the `Host` column of `SHOW PROCESSLIST` and of
`information_schema.processlist`;
- the audit log's `client_ip` (`AuditLogHelper` reads
`ctx.getClientIP()`);
- the `kill query from ...` and connection-timeout warnings in `fe.log`.
With every Flight session showing the same placeholder, there was no way
to tell where one came
from — which client to talk to, which one to `KILL`.
The address is already resolved and already on the session:
`FlightRemoteIpServerStreamTracer`
captures it from the gRPC transport when the bearer token is issued, and
`FlightSessionsManager.buildConnectContext()` stores it via
`setRemoteIP()`. This PR reports that,
falling back to the tracer's `0.0.0.0` sentinel when the address could
not be resolved. The two
placeholder methods on `FlightSqlChannel` have no callers left and are
removed.
Only the address is reported, not the `host:port` pair MySQL reports: a
Flight session has no stable
peer port, because each gRPC call of a session may arrive on its own
connection. A port that changes
under the operator is worse than no port.
---
.../auth2/FlightRemoteIpServerStreamTracer.java | 2 +-
.../arrowflight/results/FlightSqlChannel.java | 10 ---
.../sessions/FlightSqlConnectContext.java | 9 +-
.../FlightSqlConnectContextClientIpTest.java | 95 ++++++++++++++++++++++
.../test_processlist_client_ip.groovy | 49 +++++++++++
5 files changed, 152 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightRemoteIpServerStreamTracer.java
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightRemoteIpServerStreamTracer.java
index 5f5deee49bc..8c756c599a3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightRemoteIpServerStreamTracer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/auth2/FlightRemoteIpServerStreamTracer.java
@@ -33,7 +33,7 @@ import java.net.SocketAddress;
* seed the remote IP into the gRPC Context for Basic credential validation.
*/
public class FlightRemoteIpServerStreamTracer extends ServerStreamTracer {
- static final String UNKNOWN_REMOTE_IP = "0.0.0.0";
+ public static final String UNKNOWN_REMOTE_IP = "0.0.0.0";
private static final Context.Key<RemoteIpHolder> REMOTE_IP_CONTEXT_KEY =
Context.key("doris.arrow.flight.remote_ip");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
index 2781994dfa1..678dd7bb69f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java
@@ -52,16 +52,6 @@ public class FlightSqlChannel {
allocator = new RootAllocator(Long.MAX_VALUE);
}
- // TODO
- public String getRemoteIp() {
- return "0.0.0.0";
- }
-
- // TODO
- public String getRemoteHostPortString() {
- return "0.0.0.0:0";
- }
-
public void addResult(String queryId, String runningQuery, ResultSet
resultSet) {
List<Field> schemaFields = new ArrayList<>();
List<FieldVector> dataFields = new ArrayList<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContext.java
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContext.java
index ceddfaa5639..caa7a045437 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContext.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ConnectProcessor;
+import
org.apache.doris.service.arrowflight.auth2.FlightRemoteIpServerStreamTracer;
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
import org.apache.doris.thrift.TResultSinkType;
import org.apache.doris.thrift.TStatusCode;
@@ -57,7 +58,7 @@ public class FlightSqlConnectContext extends ConnectContext {
@Override
public String getClientIP() {
- return flightSqlChannel.getRemoteHostPortString();
+ return getRemoteHostPortString();
}
@Override
@@ -90,7 +91,11 @@ public class FlightSqlConnectContext extends ConnectContext {
@Override
public String getRemoteHostPortString() {
- return getFlightSqlChannel().getRemoteHostPortString();
+ // An Arrow Flight SQL session has no MysqlChannel. The client address
is captured when the
+ // bearer token is issued (FlightRemoteIpServerStreamTracer) and kept
on the context. There is
+ // no stable peer port to report: every gRPC call of a session may
arrive on its own connection.
+ return Strings.isNullOrEmpty(getRemoteIP())
+ ? FlightRemoteIpServerStreamTracer.UNKNOWN_REMOTE_IP :
getRemoteIP();
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContextClientIpTest.java
b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContextClientIpTest.java
new file mode 100644
index 00000000000..b82463da921
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContextClientIpTest.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service.arrowflight.sessions;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.system.SystemInfoService;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * An Arrow Flight SQL session reported 0.0.0.0 everywhere a client address is
shown - the Host column
+ * of SHOW PROCESSLIST and information_schema.processlist, the audit log's
client_ip, and the
+ * kill/timeout warnings - because FlightSqlChannel had those accessors
stubbed out. The real address
+ * is resolved when the bearer token is issued
(FlightRemoteIpServerStreamTracer) and stored on the
+ * context, so that is what the session reports.
+ */
+public class FlightSqlConnectContextClientIpTest {
+ private static final String CLIENT_IP = "10.26.20.3";
+ private static final String UNKNOWN_IP = "0.0.0.0";
+
+ private boolean savedRunningUnitTest;
+
+ @BeforeEach
+ public void setUp() {
+ savedRunningUnitTest = FeConstants.runningUnitTest;
+ // ConnectContext.init() registers the session with Env unless running
as a unit test.
+ FeConstants.runningUnitTest = true;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ FeConstants.runningUnitTest = savedRunningUnitTest;
+ }
+
+ @Test
+ public void testProcesslistAndAuditSeeTheAuthenticatedClientIp() {
+ try (MockedStatic<Env> mockedEnv = mockSelfNode()) {
+ FlightSqlConnectContext ctx = new
FlightSqlConnectContext("test-peer-identity");
+
ctx.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp("alice",
"%"));
+ ctx.setRemoteIP(CLIENT_IP);
+
+ Assertions.assertEquals(CLIENT_IP, ctx.getRemoteHostPortString());
+ // AuditLogHelper and LineageUtils read the address through
getClientIP().
+ Assertions.assertEquals(CLIENT_IP, ctx.getClientIP());
+
+ List<String> row = ctx.toThreadInfo(false).toRow(1,
System.currentTimeMillis(), Optional.empty());
+ // Host is the fourth column of SHOW PROCESSLIST /
information_schema.processlist.
+ Assertions.assertEquals(CLIENT_IP, row.get(3));
+ }
+ }
+
+ @Test
+ public void testFallsBackWhenTheAddressWasNotResolved() {
+ try (MockedStatic<Env> mockedEnv = mockSelfNode()) {
+ FlightSqlConnectContext ctx = new
FlightSqlConnectContext("test-peer-identity");
+
+ Assertions.assertEquals(UNKNOWN_IP, ctx.getRemoteHostPortString());
+ ctx.setRemoteIP("");
+ Assertions.assertEquals(UNKNOWN_IP, ctx.getRemoteHostPortString());
+ }
+ }
+
+ private static MockedStatic<Env> mockSelfNode() {
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new
SystemInfoService.HostInfo("127.0.0.1", 9010));
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ return mockedEnv;
+ }
+}
diff --git
a/regression-test/suites/arrow_flight_sql_p0/test_processlist_client_ip.groovy
b/regression-test/suites/arrow_flight_sql_p0/test_processlist_client_ip.groovy
new file mode 100644
index 00000000000..248269740a2
--- /dev/null
+++
b/regression-test/suites/arrow_flight_sql_p0/test_processlist_client_ip.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// An Arrow Flight SQL session used to report the placeholder 0.0.0.0:0 as its
client address,
+// which made SHOW PROCESSLIST, information_schema.processlist and the audit
log useless for
+// telling Flight sessions apart. The address is resolved when the bearer
token is issued, so
+// the session reports that.
+suite("test_processlist_client_ip", "arrow_flight_sql") {
+ // The first column marks the row of the connection running the statement.
+ def processList = arrow_flight_sql """SHOW PROCESSLIST"""
+ def ownRow = processList.find { "${it[0]}" == "Yes" }
+ assertNotNull(ownRow, "the Flight session does not appear in its own SHOW
PROCESSLIST")
+
+ def connectionId = "${ownRow[1]}"
+ def host = "${ownRow[3]}"
+ logger.info("arrow flight session: id=${connectionId}, host=${host}")
+
+ assertFalse(host.isEmpty(), "SHOW PROCESSLIST reports an empty host for
the Flight session")
+ assertFalse(host.startsWith("0.0.0.0"),
+ "SHOW PROCESSLIST still reports the placeholder host for the
Flight session: ${host}")
+
+ // information_schema.processlist is what monitoring actually queries, and
it is served from a
+ // different code path (the BE schema scanner). It must show the same
address. Ask over the
+ // MySQL protocol, so that the address is read by a session other than the
Flight one.
+ //
+ // The scanner asks every registered frontend for its session list, and a
frontend registered
+ // under more than one of its host's addresses answers once per
registration, so a session can
+ // come back as several identical rows. Compare the set of addresses
instead of the row count:
+ // every row for this session must report the address the Flight session
reported above.
+ def hosts = jdbc_sql """SELECT DISTINCT Host FROM
information_schema.processlist WHERE Id = ${connectionId}"""
+ assertEquals(1, hosts.size(),
+ "information_schema.processlist does not report a single host for
Flight session "
+ + "${connectionId}: ${hosts}")
+ assertEquals(host, "${hosts[0][0]}")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
