[
https://issues.apache.org/jira/browse/HADOOP-17618?focusedWorklogId=587315&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-587315
]
ASF GitHub Bot logged work on HADOOP-17618:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 22/Apr/21 15:33
Start Date: 22/Apr/21 15:33
Worklog Time Spent: 10m
Work Description: steveloughran commented on a change in pull request
#2845:
URL: https://github.com/apache/hadoop/pull/2845#discussion_r618504576
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
##########
@@ -227,11 +229,30 @@ public String getLogString() {
.append(" m=")
.append(method)
.append(" u=")
- .append(getSignatureMaskedEncodedUrl());
+ .append(getMaskedEncodedUrl());
return sb.toString();
}
+ public String getMaskedUrl() {
+ if (!shouldMask) {
+ return url.toString();
+ }
+ if (this.maskedUrl != null) {
Review comment:
remove the this. prefix; not needed
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
##########
@@ -73,6 +98,63 @@ public static String generateUniqueTestPath() {
return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId +
"/test";
}
+ public static String maskUrlQueryParameters(List<NameValuePair> keyValueList,
+ HashSet<String> queryParamsForFullMask,
+ HashSet<String> queryParamsForPartialMask) {
+ return maskUrlQueryParameters(keyValueList, queryParamsForFullMask,
+ queryParamsForPartialMask, 256);
+ }
+
+ public static String maskUrlQueryParameters(List<NameValuePair> keyValueList,
+ HashSet<String> queryParamsForFullMask,
+ HashSet<String> queryParamsForPartialMask, int queryLen) {
+ StringBuilder maskedUrl = new StringBuilder(queryLen);
+ for (NameValuePair keyValuePair : keyValueList) {
+ String key = keyValuePair.getName();
+ if (key.isEmpty()) {
+ throw new IllegalArgumentException("Query param key should not be
empty");
+ }
+ String value = keyValuePair.getValue();
+ maskedUrl.append(key);
+ maskedUrl.append(EQUAL);
+ if (value != null && !value.isEmpty()) { //no mask
+ if (queryParamsForFullMask.contains(key)) {
+ maskedUrl.append(FULL_MASK);
+ } else if (queryParamsForPartialMask.contains(key)) {
+ int valueLen = value.length();
+ int maskedLen = valueLen - Math.min(4, valueLen);
+ maskedUrl.append(StringUtils.repeat(CHAR_MASK, maskedLen));
+ maskedUrl.append(value, maskedLen, valueLen);
+ } else {
+ maskedUrl.append(value);
+ }
+ }
+ maskedUrl.append(AND_MARK);
+ }
+ maskedUrl.deleteCharAt(maskedUrl.length() - 1);
+ return maskedUrl.toString();
+ }
+
+ public static String encodedUrlStr(String url) {
+ try {
+ return URLEncoder.encode(url, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ return "https%3A%2F%2Ffailed%2Fto%2Fencode%2Furl";
Review comment:
heh. nice.
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
##########
@@ -18,14 +18,39 @@
package org.apache.hadoop.fs.azurebfs.utils;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.AND_MARK;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EQUAL;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SAOID;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SIGNATURE;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SKOID;
+import static
org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_SUOID;
+
/**
* Utility class to help with Abfs url transformation to blob urls.
*/
public final class UriUtils {
private static final String ABFS_URI_REGEX =
"[^.]+\\.dfs\\.(preprod\\.){0,1}core\\.windows\\.net";
private static final Pattern ABFS_URI_PATTERN =
Pattern.compile(ABFS_URI_REGEX);
+ private static final HashSet<String> FULL_MASK_PARAM_KEYS = new HashSet<>(
Review comment:
use `Set<String>` as the type of variable
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
##########
@@ -73,6 +98,63 @@ public static String generateUniqueTestPath() {
return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId +
"/test";
}
+ public static String maskUrlQueryParameters(List<NameValuePair> keyValueList,
+ HashSet<String> queryParamsForFullMask,
+ HashSet<String> queryParamsForPartialMask) {
+ return maskUrlQueryParameters(keyValueList, queryParamsForFullMask,
+ queryParamsForPartialMask, 256);
+ }
+
+ public static String maskUrlQueryParameters(List<NameValuePair> keyValueList,
+ HashSet<String> queryParamsForFullMask,
+ HashSet<String> queryParamsForPartialMask, int queryLen) {
+ StringBuilder maskedUrl = new StringBuilder(queryLen);
+ for (NameValuePair keyValuePair : keyValueList) {
+ String key = keyValuePair.getName();
+ if (key.isEmpty()) {
+ throw new IllegalArgumentException("Query param key should not be
empty");
+ }
+ String value = keyValuePair.getValue();
+ maskedUrl.append(key);
+ maskedUrl.append(EQUAL);
+ if (value != null && !value.isEmpty()) { //no mask
+ if (queryParamsForFullMask.contains(key)) {
+ maskedUrl.append(FULL_MASK);
+ } else if (queryParamsForPartialMask.contains(key)) {
+ int valueLen = value.length();
+ int maskedLen = valueLen - Math.min(4, valueLen);
Review comment:
make this 4 a constant too, allows for different levels of masking to be
used in future.
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
##########
@@ -73,6 +98,63 @@ public static String generateUniqueTestPath() {
return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId +
"/test";
}
+ public static String maskUrlQueryParameters(List<NameValuePair> keyValueList,
+ HashSet<String> queryParamsForFullMask,
Review comment:
and have these methods take Set<> too. Just to hide the detail that
HashSet is used
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 587315)
Time Spent: 5h (was: 4h 50m)
> ABFS: Partially obfuscate SAS object IDs in Logs
> ------------------------------------------------
>
> Key: HADOOP-17618
> URL: https://issues.apache.org/jira/browse/HADOOP-17618
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.3.1
> Reporter: Sumangala Patki
> Assignee: Sumangala Patki
> Priority: Major
> Labels: pull-request-available
> Time Spent: 5h
> Remaining Estimate: 0h
>
> Delegation SAS tokens are created using various parameters for specifying
> details such as permissions and validity. The requests are logged, along with
> values of all the query parameters. This change will partially mask values
> logged for the following object IDs representing the security principal:
> skoid, saoid, suoid
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
