This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new e9bbfcd9cee HDFS-17940. WebHDFS to log at TRACE better (#8565) (#8581)
e9bbfcd9cee is described below
commit e9bbfcd9cee8170ee7d5531d46e08273a7d54435
Author: Steve Loughran <[email protected]>
AuthorDate: Thu Jul 2 10:50:18 2026 +0100
HDFS-17940. WebHDFS to log at TRACE better (#8565) (#8581)
Contains content generated by OpenAI Codex
Contributed by drewrukin
---
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 5 ++
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 61 ++++++++++++++++++--
.../hadoop/hdfs/web/TestWebHdfsUrlLogging.java | 66 ++++++++++++++++++++++
3 files changed, 127 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 1c8164de65d..4227ab628f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -123,6 +123,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index a8bd95b32a2..8860a57ab33 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -46,6 +46,7 @@
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -599,7 +600,9 @@ private URL getNamenodeURL(String path, String query)
throws IOException {
InetSocketAddress nnAddr = getCurrentNNAddr();
final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
nnAddr.getPort(), path + '?' + query);
- LOG.trace("url={}", url);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("url={}", getMaskedUrlForLog(url));
+ }
return url;
}
@@ -641,7 +644,9 @@ URL toUrl(final HttpOpParam.Op op, final Path fspath,
+ Param.toSortedString("&", getAuthParameters(op))
+ Param.toSortedString("&", parameters);
final URL url = getNamenodeURL(path, query);
- LOG.trace("url={}", url);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("url={}", getMaskedUrlForLog(url));
+ }
return url;
}
@@ -1696,6 +1701,52 @@ protected HttpURLConnection connect(final long offset,
}
private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
+ private static final String LOG_MASKED_PARAM_VALUE = "XXXXX";
+ private static final String[] LOG_MASKED_PARAM_PREFIXES = {
+ DelegationParam.NAME + "=",
+ TokenArgumentParam.NAME + "="
+ };
+
+ @VisibleForTesting
+ static String getMaskedUrlForLog(final URL url) {
+ final String query = url.getQuery();
+ if (query == null) {
+ return url.toString();
+ }
+ final String lower = query.toLowerCase(Locale.ROOT);
+ boolean mask = false;
+ for (String prefix : LOG_MASKED_PARAM_PREFIXES) {
+ if (lower.startsWith(prefix) || lower.contains("&" + prefix)) {
+ mask = true;
+ break;
+ }
+ }
+ if (!mask) {
+ return url.toString();
+ }
+
+ final StringBuilder b = new StringBuilder("?");
+ for (final StringTokenizer st = new StringTokenizer(query, "&");
+ st.hasMoreTokens();) {
+ if (b.length() > 1) {
+ b.append('&');
+ }
+ b.append(maskQueryTokenForLog(st.nextToken()));
+ }
+
+ final String urlStr = url.toString();
+ return urlStr.substring(0, urlStr.indexOf('?')) + b;
+ }
+
+ private static String maskQueryTokenForLog(final String token) {
+ final String lower = token.toLowerCase(Locale.ROOT);
+ for (String prefix : LOG_MASKED_PARAM_PREFIXES) {
+ if (lower.startsWith(prefix) && token.length() > prefix.length()) {
+ return token.substring(0, prefix.length()) + LOG_MASKED_PARAM_VALUE;
+ }
+ }
+ return token;
+ }
/** Remove offset parameter, if there is any, from the url */
static URL removeOffsetParam(final URL url) throws MalformedURLException {
@@ -1703,7 +1754,7 @@ static URL removeOffsetParam(final URL url) throws
MalformedURLException {
if (query == null) {
return url;
}
- final String lower = StringUtils.toLowerCase(query);
+ final String lower = query.toLowerCase(Locale.ROOT);
if (!lower.startsWith(OFFSET_PARAM_PREFIX)
&& !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
return url;
@@ -1714,7 +1765,7 @@ static URL removeOffsetParam(final URL url) throws
MalformedURLException {
for(final StringTokenizer st = new StringTokenizer(query, "&");
st.hasMoreTokens();) {
final String token = st.nextToken();
- if (!StringUtils.toLowerCase(token).startsWith(OFFSET_PARAM_PREFIX)) {
+ if (!token.toLowerCase(Locale.ROOT).startsWith(OFFSET_PARAM_PREFIX)) {
if (b == null) {
b = new StringBuilder("?").append(token);
} else {
@@ -2632,7 +2683,7 @@ InputStream initializeInputStream(HttpURLConnection conn)
final String cl = conn.getHeaderField(HttpHeaders.CONTENT_LENGTH);
InputStream inStream = conn.getInputStream();
if (LOG.isDebugEnabled()) {
- LOG.debug("open file: " + conn.getURL());
+ LOG.debug("open file: {}", getMaskedUrlForLog(conn.getURL()));
}
if (cl != null) {
long streamLength = Long.parseLong(cl);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrlLogging.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrlLogging.java
new file mode 100644
index 00000000000..99afe34b0ce
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrlLogging.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.net.URL;
+
+import org.junit.Test;
+
+public class TestWebHdfsUrlLogging {
+
+ @Test
+ public void testMaskedUrlForLog() throws IOException {
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc";)))
+ .isEqualTo("http://test/Abc";);
+
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc?op=OPEN&length=99";)))
+ .isEqualTo("http://test/Abc?op=OPEN&length=99";);
+
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc?delegation=secret&op=OPEN";)))
+ .isEqualTo("http://test/Abc?delegation=XXXXX&op=OPEN";);
+
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc?op=OPEN&Token=secret&length=99";)))
+ .isEqualTo("http://test/Abc?op=OPEN&Token=XXXXX&length=99";);
+
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc?token=first&op=OPEN&token=second";)))
+ .isEqualTo("http://test/Abc?token=XXXXX&op=OPEN&token=XXXXX";);
+
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(
+ new URL("http://test/Abc?token=&delegation=&op=OPEN";)))
+ .isEqualTo("http://test/Abc?token=&delegation=&op=OPEN";);
+
+ final String similarNames =
+ "http://test/Abc?mytoken=secret&delegationx=secret&op=OPEN";;
+ assertThat(WebHdfsFileSystem.getMaskedUrlForLog(new URL(similarNames)))
+ .isEqualTo(similarNames);
+
+ final String masked = WebHdfsFileSystem.getMaskedUrlForLog(new URL(
+ "http://test/Abc?op=OPEN&delegation=secret&token=another";));
+ assertThat(masked)
+ .doesNotContain("secret")
+ .doesNotContain("another");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
