This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new d34fa34817e branch-4.0: [improvement](fe) add fe_meta_auth_token for
FE meta-service internal HTTP auth (#66095)
d34fa34817e is described below
commit d34fa34817ec1a795956c1fc1a32dc29c2cc844b
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Jul 28 09:30:51 2026 +0800
branch-4.0: [improvement](fe) add fe_meta_auth_token for FE meta-service
internal HTTP auth (#66095)
This supersedes #63782 (rebased onto latest master with a cleaner,
switch-free design).
---
.../main/java/org/apache/doris/common/Config.java | 14 +-
.../java/org/apache/doris/common/ConfigBase.java | 20 +-
.../java/org/apache/doris/common/ConfigTest.java | 58 ++++
.../org/apache/doris/common/util/HttpURLUtil.java | 15 +
.../org/apache/doris/httpv2/meta/MetaService.java | 77 +++++-
.../doris/httpv2/rest/manager/NodeAction.java | 11 +-
.../java/org/apache/doris/master/MetaHelper.java | 2 +-
.../apache/doris/common/util/HttpURLUtilTest.java | 93 +++++++
.../apache/doris/httpv2/meta/MetaServiceTest.java | 305 +++++++++++++++++++++
.../suites/meta_action_p0/test_dump_image.groovy | 1 +
10 files changed, 580 insertions(+), 16 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 5b0d5245b47..671f707b356 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -536,7 +536,7 @@ public class Config extends ConfigBase {
+ "start at first time. You can also specify one."})
public static int cluster_id = -1;
- @ConfField(description = {"集群 token，用于内部认证。",
+ @ConfField(sensitive = true, description = {"集群 token，用于内部认证。",
"Cluster token used for internal authentication."})
public static String auth_token = "";
@@ -911,6 +911,18 @@ public class Config extends ConfigBase {
// check token when download image file.
@ConfField public static boolean enable_token_check = true;
+ @ConfField(sensitive = true, description = {"Cluster token for FE
meta-service internal HTTP authentication. "
+ + "When set (non-empty), FE meta-service endpoints (such as
image/role/check/put/journal_id) "
+ + "additionally require the caller to present a matching token
header, on top of the existing "
+ + "node-host check. Empty (default) keeps the legacy behavior of
node-host check only, so "
+ + "existing clusters and rolling upgrades are unaffected. Must be
identical on all FEs and "
+ + "provisioned in fe.conf before enabling, otherwise FEs will
reject each other.",
+ "FE meta-service 内部 HTTP 鉴权使用的集群 token。设置(非空)后,meta-service 端点(如 "
+ + "image/role/check/put/journal_id)在原有 node-host 校验之上,额外要求调用方携带匹配的
token 头。"
+ + "为空(默认)时维持仅 node-host 校验的旧行为,存量集群与滚动升级不受影响。必须在所有 FE 上取值一致,"
+ + "并在启用前写入 fe.conf,否则 FE 之间会互相拒绝。"})
+ public static String fe_meta_auth_token = "";
+
/**
* Set to true if you deploy Palo using thirdparty deploy manager
* Valid options are:
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
index d6b9b2b3615..177603f0fc2 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
@@ -52,6 +52,10 @@ public class ConfigBase {
boolean masterOnly() default false;
+ // If true, the value is a secret (e.g. a token or password) and is
masked in every
+ // config dump API (Config.dump / getConfigInfo), so it is never
returned in plaintext.
+ boolean sensitive() default false;
+
String comment() default "";
VariableAnnotation varType() default VariableAnnotation.NONE;
@@ -162,13 +166,26 @@ public class ConfigBase {
setFields(props, isLdapConfig);
}
+ // Placeholder returned instead of a sensitive config's real value in any
dump API.
+ public static final String SENSITIVE_CONF_MASK = "********";
+
+ // Mask the value of a sensitive config (a non-empty secret) so it is
never dumped in plaintext.
+ // An empty value is left as-is: it reveals nothing and keeps "unset"
visible.
+ private static String maskIfSensitive(Field field, String value) {
+ ConfField anno = field.getAnnotation(ConfField.class);
+ if (anno != null && anno.sensitive() && !Strings.isNullOrEmpty(value))
{
+ return SENSITIVE_CONF_MASK;
+ }
+ return value;
+ }
+
public static HashMap<String, String> dump() {
HashMap<String, String> map = new HashMap<>();
Field[] fields = confClass.getFields();
for (Field f : fields) {
ConfField anno = f.getAnnotation(ConfField.class);
if (anno != null) {
- map.put(f.getName(), getConfValue(f));
+ map.put(f.getName(), maskIfSensitive(f, getConfValue(f)));
}
}
return map;
@@ -412,6 +429,7 @@ public class ConfigBase {
if (confKey.equals("sys_log_dir") &&
Strings.isNullOrEmpty(value)) {
value = System.getenv("DORIS_HOME") + "/log";
}
+ value = maskIfSensitive(f, value);
config.add(value);
config.add(f.getType().getSimpleName());
config.add(String.valueOf(confField.mutable()));
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
index 0d305298270..d48d7ae8ed3 100644
--- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java
@@ -23,6 +23,8 @@ import org.junit.Test;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
public class ConfigTest {
@BeforeClass
@@ -34,6 +36,62 @@ public class ConfigTest {
config.init(tempFile.toAbsolutePath().toString());
}
+ // A sensitive config (fe_meta_auth_token) must never be dumped in
plaintext by any config
+ // API: both Config.dump() and ConfigBase.getConfigInfo() return the mask
instead of the value.
+ @Test
+ public void testSensitiveConfigIsMaskedWhenSet() {
+ String old = Config.fe_meta_auth_token;
+ try {
+ Config.fe_meta_auth_token = "super-secret-token";
+
+ Map<String, String> dumped = ConfigBase.dump();
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
dumped.get("fe_meta_auth_token"));
+
+ String value = configInfoValue("fe_meta_auth_token");
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, value);
+ } finally {
+ Config.fe_meta_auth_token = old;
+ }
+ }
+
+ // The legacy cluster secret auth_token is also marked sensitive, so it is
masked by every
+ // config dump API too (it leaks through /rest/v1/config/fe otherwise).
+ @Test
+ public void testAuthTokenIsMaskedWhenSet() {
+ String old = Config.auth_token;
+ try {
+ Config.auth_token = "super-secret-auth-token";
+
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
ConfigBase.dump().get("auth_token"));
+ Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK,
configInfoValue("auth_token"));
+ } finally {
+ Config.auth_token = old;
+ }
+ }
+
+ // An empty sensitive config is left as-is (no secret to hide), so "unset"
stays visible.
+ @Test
+ public void testEmptySensitiveConfigIsNotMasked() {
+ String old = Config.fe_meta_auth_token;
+ try {
+ Config.fe_meta_auth_token = "";
+
+ Assert.assertEquals("",
ConfigBase.dump().get("fe_meta_auth_token"));
+ Assert.assertEquals("", configInfoValue("fe_meta_auth_token"));
+ } finally {
+ Config.fe_meta_auth_token = old;
+ }
+ }
+
+ private static String configInfoValue(String key) {
+ for (List<String> row : ConfigBase.getConfigInfo(null)) {
+ if (row.get(0).equals(key)) {
+ return row.get(1);
+ }
+ }
+ throw new IllegalStateException("config not found: " + key);
+ }
+
@Test
public void testSetEmptyArray() throws ConfigException {
ConfigBase.setMutableConfig("s3_load_endpoint_white_list", "a,b,c");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
index 70b3e68450a..67948e8b31a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java
@@ -19,8 +19,11 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.security.SecurityChecker;
+import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.meta.MetaBaseAction;
import org.apache.doris.system.SystemInfoService.HostInfo;
+import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import java.io.IOException;
@@ -40,6 +43,10 @@ public class HttpURLUtil {
HostInfo selfNode = Env.getServingEnv().getSelfNode();
conn.setRequestProperty(Env.CLIENT_NODE_HOST_KEY,
selfNode.getHost());
conn.setRequestProperty(Env.CLIENT_NODE_PORT_KEY,
selfNode.getPort() + "");
+ String token = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(token)) {
+ conn.setRequestProperty(MetaBaseAction.TOKEN, token);
+ }
return conn;
} catch (Exception e) {
throw new IOException(e);
@@ -55,7 +62,15 @@ public class HttpURLUtil {
HostInfo selfNode = Env.getServingEnv().getSelfNode();
headers.put(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost());
headers.put(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + "");
+ String token = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(token)) {
+ headers.put(MetaBaseAction.TOKEN, token);
+ }
return headers;
}
+ public static int getHttpPort() {
+ return Config.enable_https ? Config.https_port : Config.http_port;
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
index 6a8bd71faca..13f0e6a2ddc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java
@@ -20,14 +20,18 @@ package org.apache.doris.httpv2.meta;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.util.HttpURLUtil;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.ha.FrontendNodeType;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
+import org.apache.doris.httpv2.exception.UnauthorizedException;
import org.apache.doris.httpv2.rest.RestBaseController;
import org.apache.doris.master.MetaHelper;
+import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.MetaCleaner;
import org.apache.doris.persist.Storage;
import org.apache.doris.persist.StorageInfo;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.Frontend;
import com.google.common.base.Strings;
@@ -55,33 +59,76 @@ public class MetaService extends RestBaseController {
private File imageDir = MetaHelper.getMasterImageDir();
- private boolean isFromValidFe(String clientHost, String clientPortStr) {
+ private Frontend getValidFe(String clientHost, String clientPortStr) {
Integer clientPort;
try {
clientPort = Integer.valueOf(clientPortStr);
} catch (Exception e) {
LOG.warn("get clientPort error. clientPortStr: {}", clientPortStr,
e.getMessage());
- return false;
+ return null;
}
Frontend fe = Env.getCurrentEnv().checkFeExist(clientHost, clientPort);
if (fe == null) {
LOG.warn("request is not from valid FE. client: {}, {}",
clientHost, clientPortStr);
- return false;
}
- return true;
+ return fe;
}
private void checkFromValidFe(HttpServletRequest request)
- throws InvalidClientException {
+ throws UnauthorizedException {
String clientHost = request.getHeader(Env.CLIENT_NODE_HOST_KEY);
String clientPort = request.getHeader(Env.CLIENT_NODE_PORT_KEY);
- if (!isFromValidFe(clientHost, clientPort)) {
- throw new InvalidClientException("invalid client host: " +
clientHost + ":" + clientPort
- + ", request from " + request.getRemoteHost());
+ Frontend fe = getValidFe(clientHost, clientPort);
+ if (fe == null) {
+ throw unauthorized(clientHost, clientPort, request);
+ }
+
+ // If a cluster meta auth token is configured, additionally require
the request to
+ // carry a matching token. An empty token keeps the legacy
node-host-only behavior,
+ // so existing clusters and rolling upgrades are unaffected.
+ String clusterToken = Config.fe_meta_auth_token;
+ if (!Strings.isNullOrEmpty(clusterToken)) {
+ String requestToken = request.getHeader(MetaBaseAction.TOKEN);
+ if (!clusterToken.equals(requestToken)) {
+ // Log a masked prefix of both tokens so token-rotation issues
are diagnosable
+ // (e.g. expected "abc***" vs actual "<empty>" means the peer
sent no token),
+ // while never revealing the full secret.
+ LOG.warn("reject meta request with invalid token. client: {},
{}, request from: {}, "
+ + "expected: {}, actual: {}",
+ clientHost, clientPort, request.getRemoteAddr(),
+ maskToken(clusterToken), maskToken(requestToken));
+ throw unauthorized(clientHost, clientPort, request);
+ }
}
}
+ private UnauthorizedException unauthorized(String clientHost, String
clientPort, HttpServletRequest request) {
+ return new UnauthorizedException("invalid client host: " + clientHost
+ ":" + clientPort
+ + ", request from " + request.getRemoteAddr());
+ }
+
+ // Minimum token length required before we reveal a masked prefix in logs.
Shorter tokens would
+ // leak too large a fraction of the secret, so they are hidden entirely
with only a length hint.
+ private static final int MIN_TOKEN_LEN_FOR_PREFIX = 8;
+ private static final int TOKEN_PREFIX_LEN = 3;
+
+ /**
+ * Masks a token for logging: reveals only a short leading prefix (e.g.
"abc***") so that a
+ * token mismatch is diagnosable during rotation, while never logging the
full secret. Empty
+ * tokens and tokens too short to safely show a prefix are hidden.
+ */
+ private static String maskToken(String token) {
+ if (Strings.isNullOrEmpty(token)) {
+ return "<empty>";
+ }
+ if (token.length() < MIN_TOKEN_LEN_FOR_PREFIX) {
+ // Too short to reveal any prefix without leaking a large fraction
of the secret.
+ return "<hidden, token length " + token.length() + " < " +
MIN_TOKEN_LEN_FOR_PREFIX + ">";
+ }
+ return token.substring(0, TOKEN_PREFIX_LEN) + "***";
+ }
+
@RequestMapping(path = "/image", method = RequestMethod.GET)
public Object image(HttpServletRequest request, HttpServletResponse
response) {
checkFromValidFe(request);
@@ -149,6 +196,12 @@ public class MetaService extends RestBaseController {
if (port < 0 || port > 65535) {
return ResponseEntityBuilder.badRequest("port is invalid. The port
number is between 0-65535");
}
+ // The master pushes image using HttpURLUtil.getHttpPort() (https_port
when enable_https=true,
+ // otherwise http_port), so the expected port must follow the same
rule to stay consistent.
+ int expectedPort = HttpURLUtil.getHttpPort();
+ if (port != expectedPort) {
+ return ResponseEntityBuilder.badRequest("port must be FE HTTP
port: " + expectedPort);
+ }
String versionStr = request.getParameter(VERSION);
if (Strings.isNullOrEmpty(versionStr)) {
@@ -242,9 +295,11 @@ public class MetaService extends RestBaseController {
@RequestMapping(value = "/dump", method = RequestMethod.GET)
public Object dump(HttpServletRequest request, HttpServletResponse
response) throws DdlException {
- if (Config.enable_all_http_auth) {
- executeCheckPassword(request, response);
- }
+ // /dump triggers a full metadata image dump (takes catalog/db/table
locks and writes an
+ // image file), so it must be ADMIN-gated. executeCheckPassword only
authenticates the
+ // caller; enforce the ADMIN privilege explicitly, matching other
metadata/debug operations.
+ executeCheckPassword(request, response);
+ checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(),
PrivPredicate.ADMIN);
/*
* Before dump, we acquired the catalog read lock and all databases'
read lock and all
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
index 4067864dc30..12f4fd46fc5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java
@@ -256,8 +256,13 @@ public class NodeAction extends RestBaseController {
*/
@RequestMapping(path = "/config", method = RequestMethod.GET)
public Object config(HttpServletRequest request, HttpServletResponse
response) {
+ // This endpoint lists all FE config, matching the SQL "SHOW FRONTEND
CONFIG", which
+ // requires ADMIN. Use an unconditional ADMIN check: checkAdminAuth
only enforces the
+ // privilege when enable_all_http_auth is true, so it would be a no-op
by default.
+ // Sensitive config values (e.g. fe_meta_auth_token) are additionally
masked by ConfigBase,
+ // so they are never returned in plaintext even to an admin.
executeCheckPassword(request, response);
- checkDbAuth(ConnectContext.get().getCurrentUserIdentity(),
InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT);
+ checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(),
PrivPredicate.ADMIN);
List<List<String>> configs = ConfigBase.getConfigInfo(null);
// Sort all configs by config key.
@@ -317,8 +322,10 @@ public class NodeAction extends RestBaseController {
public Object configurationInfo(HttpServletRequest request,
HttpServletResponse response,
@RequestParam(value = "type") String type,
@RequestBody(required = false) ConfigInfoRequestBody requestBody) {
+ // Reads FE/BE config via fan-out to the per-node config endpoints, so
it must be
+ // ADMIN-gated too. Unconditional check (see config() above for why
checkAdminAuth is not).
executeCheckPassword(request, response);
- checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(),
PrivPredicate.ADMIN_OR_NODE);
+ checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(),
PrivPredicate.ADMIN);
initHttpExecutor();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
index 96e73756791..553d2905649 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java
@@ -149,7 +149,7 @@ public class MetaHelper {
public static <T> ResponseBody doGet(String url, int timeout, Class<T>
clazz) throws IOException {
Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
- LOG.info("meta helper, url: {}, timeout{}, headers: {}", url, timeout,
headers);
+ LOG.info("meta helper, url: {}, timeout: {}, header names: {}", url,
timeout, headers.keySet());
String response = HttpUtils.doGet(url, headers, timeout);
return parseResponse(response, clazz);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
new file mode 100644
index 00000000000..a3f98e8f026
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.util;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
+import org.apache.doris.httpv2.meta.MetaBaseAction;
+import org.apache.doris.system.SystemInfoService.HostInfo;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.net.HttpURLConnection;
+import java.util.Map;
+
+public class HttpURLUtilTest {
+
+ @After
+ public void tearDown() {
+ Config.enable_https = false;
+ Config.http_port = 8030;
+ Config.https_port = 8050;
+ Config.fe_meta_auth_token = "";
+ }
+
+ @Test
+ public void testNodeIdentHeadersIncludeClusterToken() throws Exception {
+ Config.fe_meta_auth_token = "cluster-token";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
+
+ Assert.assertEquals("127.0.0.1",
headers.get(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertEquals("9010", headers.get(Env.CLIENT_NODE_PORT_KEY));
+ Assert.assertEquals("cluster-token",
headers.get(MetaBaseAction.TOKEN));
+ }
+ }
+
+ @Test
+ public void testNodeIdentHeadersOmitTokenWhenNotConfigured() throws
Exception {
+ Config.fe_meta_auth_token = "";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ Map<String, String> headers = HttpURLUtil.getNodeIdentHeaders();
+
+ Assert.assertEquals("127.0.0.1",
headers.get(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertFalse(headers.containsKey(MetaBaseAction.TOKEN));
+ }
+ }
+
+ @Test
+ public void testNodeIdentConnectionIncludesClusterToken() throws Exception
{
+ Config.fe_meta_auth_token = "cluster-token";
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1",
9010));
+
+ try (MockedStatic<Env> envStatic = Mockito.mockStatic(Env.class)) {
+ envStatic.when(Env::getServingEnv).thenReturn(env);
+
+ HttpURLConnection connection =
HttpURLUtil.getConnectionWithNodeIdent("http://127.0.0.1:8030/info";);
+
+ Assert.assertEquals("127.0.0.1",
connection.getRequestProperty(Env.CLIENT_NODE_HOST_KEY));
+ Assert.assertEquals("9010",
connection.getRequestProperty(Env.CLIENT_NODE_PORT_KEY));
+ Assert.assertEquals("cluster-token",
connection.getRequestProperty(MetaBaseAction.TOKEN));
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
new file mode 100644
index 00000000000..ad5569f0d0e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java
@@ -0,0 +1,305 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.meta;
+
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
+import org.apache.doris.ha.FrontendNodeType;
+import org.apache.doris.httpv2.controller.BaseController;
+import org.apache.doris.httpv2.entity.ResponseBody;
+import org.apache.doris.httpv2.exception.UnauthorizedException;
+import org.apache.doris.httpv2.rest.RestApiStatusCode;
+import org.apache.doris.master.MetaHelper;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.persist.Storage;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.system.Frontend;
+
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.springframework.http.ResponseEntity;
+
+import java.io.File;
+import java.lang.reflect.Field;
+
+public class MetaServiceTest {
+ // Value configured via Config.fe_meta_auth_token -- the cluster meta auth
token.
+ private static final String META_TOKEN = "meta-auth-token";
+ // Token persisted in the local Storage, returned by the /check endpoint.
Intentionally
+ // different from META_TOKEN to show the two are independent.
+ private static final String STORAGE_TOKEN = "storage-token";
+ private static final String BAD_TOKEN = "bad-token";
+ private static final String FE_HOST = "127.0.0.1";
+ private static final int FE_EDIT_LOG_PORT = 9010;
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private String oldMetaAuthToken;
+ private boolean oldEnableAllHttpAuth;
+ private int oldHttpPort;
+ private int oldHttpsPort;
+ private boolean oldEnableHttps;
+ private Env env;
+ private MockedStatic<Env> envStatic;
+
+ @Before
+ public void setUp() {
+ oldMetaAuthToken = Config.fe_meta_auth_token;
+ oldEnableAllHttpAuth = Config.enable_all_http_auth;
+ oldHttpPort = Config.http_port;
+ oldHttpsPort = Config.https_port;
+ oldEnableHttps = Config.enable_https;
+ // Token required by default; individual tests clear it to exercise
the legacy path.
+ Config.fe_meta_auth_token = META_TOKEN;
+ Config.enable_all_http_auth = false;
+ Config.http_port = 8030;
+ Config.https_port = 8050;
+ Config.enable_https = false;
+
+ env = Mockito.mock(Env.class);
+ envStatic = Mockito.mockStatic(Env.class);
+ envStatic.when(Env::getCurrentEnv).thenReturn(env);
+
+ Frontend frontend = new Frontend(FrontendNodeType.FOLLOWER, "fe1",
FE_HOST, FE_EDIT_LOG_PORT);
+ Mockito.when(env.checkFeExist(FE_HOST,
FE_EDIT_LOG_PORT)).thenReturn(frontend);
+
Mockito.when(env.getImageDir()).thenReturn(temporaryFolder.getRoot().getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() {
+ Config.fe_meta_auth_token = oldMetaAuthToken;
+ Config.enable_all_http_auth = oldEnableAllHttpAuth;
+ Config.http_port = oldHttpPort;
+ Config.https_port = oldHttpsPort;
+ Config.enable_https = oldEnableHttps;
+ if (envStatic != null) {
+ envStatic.close();
+ }
+ ConnectContext.remove();
+ }
+
+ // A matching token passes even when the request originates from a proxy
address
+ // (remoteAddr differs from the registered FE host).
+ @Test
+ public void testMatchingTokenAllowsMetaRequestFromProxyAddress() throws
Exception {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST,
META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.role(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader("role",
FrontendNodeType.FOLLOWER.name());
+ }
+
+ // When no token is configured, the node-host check alone is sufficient.
This keeps
+ // existing clusters and rolling upgrades working (a peer that sends no
token is allowed).
+ @Test
+ public void testNoTokenRequiredWhenTokenNotConfigured() throws Exception {
+ Config.fe_meta_auth_token = "";
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.role(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader("role",
FrontendNodeType.FOLLOWER.name());
+ }
+
+ // /check returns the local Storage token once the request passes
authentication.
+ @Test
+ public void testCheckReturnsStorageTokenWhenAuthPasses() throws Exception {
+ Config.fe_meta_auth_token = "";
+ MetaService service = serviceWithImageDir();
+ HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Object result = service.check(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result));
+ Mockito.verify(response).setHeader(MetaBaseAction.TOKEN,
STORAGE_TOKEN);
+ }
+
+ // A wrong token is rejected even though the node-host check would pass.
+ @Test
+ public void testWrongTokenRejected() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, BAD_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ // A missing token is rejected when a cluster token is configured.
+ @Test
+ public void testMissingTokenRejectedWhenConfigured() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, null);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ // The node-host check is always enforced: an unknown FE host is rejected
even with a
+ // matching token (the token check is additive, it does not replace the
host check).
+ @Test
+ public void testUnknownHostRejectedEvenWithValidToken() {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest("192.0.2.99", "192.0.2.99",
META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.role(request, response));
+ }
+
+ @Test
+ public void testPutRejectsUnexpectedHttpPort() throws Exception {
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(request.getParameter("version")).thenReturn("100");
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.http_port
+ 1));
+
+ try (MockedStatic<MetaHelper> metaHelper =
Mockito.mockStatic(MetaHelper.class)) {
+ File partialFile = temporaryFolder.newFile("image.100.part");
+ metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(),
Mockito.any(File.class)))
+ .thenReturn(partialFile);
+
+ Object result = service.put(request, response);
+
+ Assert.assertEquals(RestApiStatusCode.BAD_REQUEST.code,
responseCode(result));
+ metaHelper.verify(() ->
MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(File.class)), Mockito.never());
+ }
+ }
+
+ // When HTTPS is enabled the master pushes image using https_port, so /put
must accept
+ // https_port. The expected port follows HttpURLUtil.getHttpPort(), not a
hard-coded http_port.
+ @Test
+ public void testPutAcceptsHttpsPortWhenHttpsEnabled() throws Exception {
+ Config.enable_https = true;
+ MetaService service = new MetaService();
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(request.getParameter("version")).thenReturn("100");
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.https_port));
+
+ try (MockedStatic<MetaHelper> metaHelper =
Mockito.mockStatic(MetaHelper.class)) {
+ File partialFile = temporaryFolder.newFile("image.100.part");
+ metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(),
Mockito.any(File.class)))
+ .thenReturn(partialFile);
+
+ Object result = service.put(request, response);
+
+ // Passes the port check and proceeds to fetch the remote image
(no BAD_REQUEST).
+ Assert.assertEquals(RestApiStatusCode.OK.code,
responseCode(result));
+ metaHelper.verify(() ->
MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(),
+ Mockito.any(File.class)), Mockito.times(1));
+ }
+ }
+
+ // /dump authenticates and then requires ADMIN. An authenticated admin
passes the privilege
+ // check and proceeds to dump.
+ @Test
+ public void testDumpAllowsAdmin() throws Exception {
+ MetaService service = Mockito.spy(new MetaService());
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
Mockito.doReturn(authInfo()).when(service).executeCheckPassword(request,
response);
+ setCurrentUser(UserIdentity.ADMIN);
+ AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ Mockito.when(accessManager.checkGlobalPriv(UserIdentity.ADMIN,
PrivPredicate.ADMIN)).thenReturn(true);
+ Mockito.when(env.dumpImage()).thenReturn(null);
+
+ service.dump(request, response);
+
+ Mockito.verify(service).executeCheckPassword(request, response);
+ Mockito.verify(env).dumpImage();
+ }
+
+ // An authenticated but non-admin user is rejected before any dump happens.
+ @Test
+ public void testDumpRejectsNonAdmin() throws Exception {
+ MetaService service = Mockito.spy(new MetaService());
+ HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ UserIdentity nonAdmin =
UserIdentity.createAnalyzedUserIdentWithIp("user", "%");
+
Mockito.doReturn(authInfo()).when(service).executeCheckPassword(request,
response);
+ setCurrentUser(nonAdmin);
+ AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ Mockito.when(accessManager.checkGlobalPriv(nonAdmin,
PrivPredicate.ADMIN)).thenReturn(false);
+
+ Assert.assertThrows(UnauthorizedException.class, () ->
service.dump(request, response));
+ Mockito.verify(env, Mockito.never()).dumpImage();
+ }
+
+ private static BaseController.ActionAuthorizationInfo authInfo() {
+ return new BaseController.ActionAuthorizationInfo();
+ }
+
+ // MetaService.dump() resolves the caller from the thread-local
ConnectContext, so the
+ // privilege check is driven by the identity installed here.
+ private static void setCurrentUser(UserIdentity userIdentity) {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setCurrentUserIdentity(userIdentity);
+ ctx.setThreadLocalInfo();
+ }
+
+ private MetaService serviceWithImageDir() throws Exception {
+ File imageDir = temporaryFolder.newFolder("image");
+ Storage storage = new Storage(12345, STORAGE_TOKEN,
imageDir.getAbsolutePath());
+ storage.writeClusterIdAndToken();
+
+ MetaService service = new MetaService();
+ Field imageDirField = MetaService.class.getDeclaredField("imageDir");
+ imageDirField.setAccessible(true);
+ imageDirField.set(service, imageDir);
+ return service;
+ }
+
+ private HttpServletRequest newRequest(String remoteAddr, String
clientHost, String token) {
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+
Mockito.when(request.getHeader(Env.CLIENT_NODE_HOST_KEY)).thenReturn(clientHost);
+
Mockito.when(request.getHeader(Env.CLIENT_NODE_PORT_KEY)).thenReturn(Integer.toString(FE_EDIT_LOG_PORT));
+
Mockito.when(request.getHeader(MetaBaseAction.TOKEN)).thenReturn(token);
+ Mockito.when(request.getRemoteAddr()).thenReturn(remoteAddr);
+ Mockito.when(request.getRemoteHost()).thenReturn(remoteAddr);
+ Mockito.when(request.getParameter("host")).thenReturn(clientHost);
+
Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(FE_EDIT_LOG_PORT));
+ return request;
+ }
+
+ private int responseCode(Object result) {
+ ResponseEntity<?> responseEntity = (ResponseEntity<?>) result;
+ ResponseBody<?> body = (ResponseBody<?>) responseEntity.getBody();
+ return body.getCode();
+ }
+}
diff --git a/regression-test/suites/meta_action_p0/test_dump_image.groovy
b/regression-test/suites/meta_action_p0/test_dump_image.groovy
index 4500503ae42..6102dfff235 100644
--- a/regression-test/suites/meta_action_p0/test_dump_image.groovy
+++ b/regression-test/suites/meta_action_p0/test_dump_image.groovy
@@ -17,6 +17,7 @@
suite("test_dump_image", "nonConcurrent") {
httpTest {
+ basicAuthorization "${context.config.jdbcUser}",
"${context.config.jdbcPassword}"
endpoint context.config.feHttpAddress
uri "/dump"
op "get"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
