[hadoop] branch trunk updated: HADOOP-17404. ABFS: Small write - Merge append and flush
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b612c31 HADOOP-17404. ABFS: Small write - Merge append and flush
b612c31 is described below
commit b612c310c26394aa406c99d8598c9cb7621df052
Author: Sneha Vijayarajan
AuthorDate: Thu Jan 7 00:13:37 2021 +0530
HADOOP-17404. ABFS: Small write - Merge append and flush
- Contributed by Sneha Vijayarajan
---
hadoop-tools/hadoop-azure/pom.xml | 2 +
.../src/config/checkstyle-suppressions.xml | 2 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 1 +
.../fs/azurebfs/constants/AbfsHttpConstants.java | 1 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 9 +
.../constants/FileSystemConfigurations.java| 1 +
.../fs/azurebfs/constants/HttpQueryParams.java | 1 +
.../services/AppendRequestParameters.java | 69 +++
.../hadoop/fs/azurebfs/services/AbfsClient.java| 47 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 67 ++-
.../azurebfs/services/AbfsOutputStreamContext.java | 11 +
.../fs/azurebfs/services/AbfsRestOperation.java| 2 +-
.../fs/azurebfs/ITestAbfsNetworkStatistics.java| 339 +++--
.../fs/azurebfs/ITestSmallWriteOptimization.java | 523 +
.../azurebfs/services/ITestAbfsOutputStream.java | 17 +-
.../fs/azurebfs/services/TestAbfsOutputStream.java | 279 ++-
17 files changed, 1030 insertions(+), 349 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml
b/hadoop-tools/hadoop-azure/pom.xml
index 2d38c9b..4176305 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -555,6 +555,7 @@
**/azurebfs/ITestAbfsReadWriteAndSeek.java
**/azurebfs/ITestAzureBlobFileSystemListStatus.java
**/azurebfs/extensions/ITestAbfsDelegationTokens.java
+
**/azurebfs/ITestSmallWriteOptimization.java
@@ -594,6 +595,7 @@
**/azurebfs/ITestAbfsReadWriteAndSeek.java
**/azurebfs/ITestAzureBlobFileSystemListStatus.java
**/azurebfs/extensions/ITestAbfsDelegationTokens.java
+
**/azurebfs/ITestSmallWriteOptimization.java
diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
index c502361..070c8c1 100644
--- a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
+++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
@@ -46,4 +46,6 @@
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]AzureBlobFileSystemStore.java"/>
+
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index b1c95d2..5a70323 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -100,6 +100,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_WRITE_BUFFER_SIZE)
private int writeBufferSize;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION,
+ DefaultValue = DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION)
+ private boolean enableSmallWriteOptimization;
+
@BooleanConfigurationValidatorAnnotation(
ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY,
DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY)
@@ -537,6 +541,10 @@ public class AbfsConfiguration{
return this.writeBufferSize;
}
+ public boolean isSmallWriteOptimizationEnabled() {
+return this.enableSmallWriteOptimization;
+ }
+
public boolean readSmallFilesCompletely() {
return this.readSmallFilesCompletely;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 869a6f9..c8dd518 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -578,6 +578,7 @@ public class AzureBlobFileSystemStore implements Closeable {
return new
AbfsOutputStreamContext(abfsConfiguration.getSasTokenRenewPeriodForStreamsInSeconds())
.withWriteBufferSi
[hadoop] branch trunk updated: HADOOP-17347. ABFS: Read optimizations
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1448add HADOOP-17347. ABFS: Read optimizations
1448add is described below
commit 1448add08fcd4a23e59eab5f75ef46fca6b1c3d1
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Sun Jan 3 00:07:10 2021 +0530
HADOOP-17347. ABFS: Read optimizations
- Contributed by Bilahari T H
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 28 ++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 2 +
.../constants/FileSystemConfigurations.java| 6 +-
.../fs/azurebfs/services/AbfsInputStream.java | 194 +--
.../azurebfs/services/AbfsInputStreamContext.java | 24 ++
.../fs/azurebfs/services/ITestAbfsInputStream.java | 256 +++
.../services/ITestAbfsInputStreamReadFooter.java | 358 +
.../ITestAbfsInputStreamSmallFileReads.java| 326 +++
9 files changed, 1175 insertions(+), 21 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 3d09a80..b1c95d2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -100,6 +100,16 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_WRITE_BUFFER_SIZE)
private int writeBufferSize;
+ @BooleanConfigurationValidatorAnnotation(
+ ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY,
+ DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY)
+ private boolean readSmallFilesCompletely;
+
+ @BooleanConfigurationValidatorAnnotation(
+ ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ,
+ DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ)
+ private boolean optimizeFooterRead;
+
@IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_READ_BUFFER_SIZE,
MinValue = MIN_BUFFER_SIZE,
MaxValue = MAX_BUFFER_SIZE,
@@ -527,6 +537,14 @@ public class AbfsConfiguration{
return this.writeBufferSize;
}
+ public boolean readSmallFilesCompletely() {
+return this.readSmallFilesCompletely;
+ }
+
+ public boolean optimizeFooterRead() {
+return this.optimizeFooterRead;
+ }
+
public int getReadBufferSize() {
return this.readBufferSize;
}
@@ -925,4 +943,14 @@ public class AbfsConfiguration{
return authority;
}
+ @VisibleForTesting
+ public void setReadSmallFilesCompletely(boolean readSmallFilesCompletely) {
+this.readSmallFilesCompletely = readSmallFilesCompletely;
+ }
+
+ @VisibleForTesting
+ public void setOptimizeFooterRead(boolean optimizeFooterRead) {
+this.optimizeFooterRead = optimizeFooterRead;
+ }
+
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index a766c62..869a6f9 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -643,6 +643,8 @@ public class AzureBlobFileSystemStore implements Closeable {
.withReadBufferSize(abfsConfiguration.getReadBufferSize())
.withReadAheadQueueDepth(abfsConfiguration.getReadAheadQueueDepth())
.withTolerateOobAppends(abfsConfiguration.getTolerateOobAppends())
+
.withReadSmallFilesCompletely(abfsConfiguration.readSmallFilesCompletely())
+.withOptimizeFooterRead(abfsConfiguration.optimizeFooterRead())
.withStreamStatistics(new AbfsInputStreamStatisticsImpl())
.withShouldReadBufferSizeAlways(
abfsConfiguration.shouldReadBufferSizeAlways())
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index cb9c0de..3e1ff80 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -56,6 +56,8 @@ public final class ConfigurationKeys {
public static final String AZURE_WRITE_MAX_REQUESTS_TO_QUEUE =
"fs.azure.write.max.requests.to.queue";
public static final String AZURE_WRITE_BUFFER_SIZE =
"fs.azure.write.request.size&quo
[hadoop] branch trunk updated: HADOOP-17407. ABFS: Fix NPE on delete idempotency flow
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5ca1ea8 HADOOP-17407. ABFS: Fix NPE on delete idempotency flow
5ca1ea8 is described below
commit 5ca1ea89b3f57017768ae4d8002f353e3d240e07
Author: Sneha Vijayarajan
AuthorDate: Sat Jan 2 23:52:10 2021 +0530
HADOOP-17407. ABFS: Fix NPE on delete idempotency flow
- Contributed by Sneha Vijayarajan
---
.../hadoop/fs/azurebfs/services/AbfsClient.java| 3 ++
.../fs/azurebfs/services/AbfsHttpOperation.java| 39 --
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 31 ++--
.../fs/azurebfs/services/TestAbfsClient.java | 46 +
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 13 +
.../hadoop/fs/azurebfs/utils/TestMockHelpers.java | 59 ++
6 files changed, 183 insertions(+), 8 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 7722c62..db2f44f 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -383,6 +383,7 @@ public class AbfsClient implements Closeable {
HttpHeaderConfigurations.LAST_MODIFIED);
if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) {
+LOG.debug("Returning success response from rename idempotency
logic");
return destStatusOp;
}
}
@@ -450,6 +451,7 @@ public class AbfsClient implements Closeable {
String fileLength = destStatusOp.getResult().getResponseHeader(
HttpHeaderConfigurations.CONTENT_LENGTH);
if (length <= Long.parseLong(fileLength)) {
+ LOG.debug("Returning success response from append blob idempotency
code");
return true;
}
}
@@ -627,6 +629,7 @@ public class AbfsClient implements Closeable {
op.getUrl(),
op.getRequestHeaders());
successOp.hardSetResult(HttpURLConnection.HTTP_OK);
+ LOG.debug("Returning success response from delete idempotency logic");
return successOp;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
index 51d0fb1..720b99b 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
@@ -86,12 +86,23 @@ public class AbfsHttpOperation implements AbfsPerfLoggable {
private long sendRequestTimeMs;
private long recvResponseTimeMs;
- public static AbfsHttpOperation getAbfsHttpOperationWithFixedResult(final
URL url,
- final String method, final int httpStatus) {
- return new AbfsHttpOperation(url, method, httpStatus);
+ public static AbfsHttpOperation getAbfsHttpOperationWithFixedResult(
+ final URL url,
+ final String method,
+ final int httpStatus) {
+AbfsHttpOperationWithFixedResult httpOp
+= new AbfsHttpOperationWithFixedResult(url, method, httpStatus);
+return httpOp;
}
- private AbfsHttpOperation(final URL url, final String method,
+ /**
+ * Constructor for FixedResult instance, avoiding connection init.
+ * @param url request url
+ * @param method Http method
+ * @param httpStatus HttpStatus
+ */
+ protected AbfsHttpOperation(final URL url,
+ final String method,
final int httpStatus) {
this.isTraceEnabled = LOG.isTraceEnabled();
this.url = url;
@@ -547,4 +558,24 @@ public class AbfsHttpOperation implements AbfsPerfLoggable
{
return this.maskedEncodedUrl;
}
+ public static class AbfsHttpOperationWithFixedResult extends
AbfsHttpOperation {
+/**
+ * Creates an instance to represent fixed results.
+ * This is used in idempotency handling.
+ *
+ * @param url The full URL including query string parameters.
+ * @param method The HTTP method (PUT, PATCH, POST, GET, HEAD, or DELETE).
+ * @param httpStatus StatusCode to hard set
+ */
+public AbfsHttpOperationWithFixedResult(final URL url,
+final String method,
+final int httpStatus) {
+ super(url, method, httpStatus);
+}
+
+@Override
+public String getResponseHeader(final String httpHeader) {
+ return "";
+}
+ }
}
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITe
[hadoop] branch trunk updated: HADOOP-17191. ABFS: Run the tests with various combinations of configurations and publish a consolidated results
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4c033ba HADOOP-17191. ABFS: Run the tests with various combinations
of configurations and publish a consolidated results
4c033ba is described below
commit 4c033bafa02855722a901def4773a6a15b214318
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Thu Dec 17 00:04:59 2020 +0530
HADOOP-17191. ABFS: Run the tests with various combinations of
configurations and publish a consolidated results
- Contributed by Bilahari T H
---
hadoop-tools/hadoop-azure/.gitignore | 5 +-
.../dev-support/testrun-scripts/runtests.sh| 50 +
.../dev-support/testrun-scripts/testsupport.sh | 241 +
.../src/site/markdown/testing_azure.md | 55 +
.../test/resources/azure-auth-keys.xml.template| 174 +++
.../hadoop-azure/src/test/resources/azure-test.xml | 4 +
6 files changed, 528 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-azure/.gitignore
b/hadoop-tools/hadoop-azure/.gitignore
index 837b481..0e17efa 100644
--- a/hadoop-tools/hadoop-azure/.gitignore
+++ b/hadoop-tools/hadoop-azure/.gitignore
@@ -1,2 +1,5 @@
.checkstyle
-bin/
\ No newline at end of file
+bin/
+src/test/resources/combinationConfigFiles
+src/test/resources/abfs-combination-test-configs.xml
+dev-support/testlogs
diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
new file mode 100755
index 000..d3d4062
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2034
+# unused variables are global in nature and used in testsupport.sh
+
+set -eo pipefail
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# shellcheck disable=SC1091
+. dev-support/testrun-scripts/testsupport.sh
+
+begin
+
+### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
+
+
+combination=HNS-OAuth
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
+"fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "true" "OAuth")
+generateconfigs
+
+combination=HNS-SharedKey
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
"fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
+generateconfigs
+
+combination=NonHNS-SharedKey
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
"fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
+generateconfigs
+
+
+### DO NOT EDIT THE LINES BELOW.
+
+runtests "$@"
diff --git
a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
new file mode 100644
index 000..1b118ae
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing perm
[hadoop] branch trunk updated: Hadoop-17413. Release elastic byte buffer pool at close
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5bf977e Hadoop-17413. Release elastic byte buffer pool at close
5bf977e is described below
commit 5bf977e6b16287d7d140dd96dad66d0fce213954
Author: Sneha Vijayarajan
AuthorDate: Tue Dec 15 10:15:37 2020 +0530
Hadoop-17413. Release elastic byte buffer pool at close
- Contributed by Sneha Vijayarajan
---
.../java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index b53b2b2..01b2fa5 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -85,7 +85,7 @@ public class AbfsOutputStream extends OutputStream implements
Syncable, StreamCa
* blocks. After the data is sent to the service, the buffer is returned
* back to the queue
*/
- private final ElasticByteBufferPool byteBufferPool
+ private ElasticByteBufferPool byteBufferPool
= new ElasticByteBufferPool();
private final Statistics statistics;
@@ -297,6 +297,7 @@ public class AbfsOutputStream extends OutputStream
implements Syncable, StreamCa
bufferIndex = 0;
closed = true;
writeOperations.clear();
+ byteBufferPool = null;
if (!threadExecutor.isShutdown()) {
threadExecutor.shutdownNow();
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16915. ABFS: Ignoring the test ITestAzureBlobFileSystemRandomRead.testRandomReadPerformance
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 64f36b9 HADOOP-16915. ABFS: Ignoring the test
ITestAzureBlobFileSystemRandomRead.testRandomReadPerformance
64f36b9 is described below
commit 64f36b9543c011ce2f1f7d1e10da0eab88a0759d
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Tue Aug 25 00:30:55 2020 +0530
HADOOP-16915. ABFS: Ignoring the test
ITestAzureBlobFileSystemRandomRead.testRandomReadPerformance
- Contributed by Bilahari T H
---
.../apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
index e5f64b5..f582763 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -23,6 +23,7 @@ import java.util.Random;
import java.util.concurrent.Callable;
import org.junit.Assume;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -412,6 +413,7 @@ public class ITestAzureBlobFileSystemRandomRead extends
}
@Test
+ @Ignore("HADOOP-16915")
public void testRandomReadPerformance() throws Exception {
Assume.assumeFalse("This test does not support namespace enabled account",
this.getFileSystem().getIsNamespaceEnabled());
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Upgrade store REST API version to 2019-12-12
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b367942 Upgrade store REST API version to 2019-12-12
b367942 is described below
commit b367942fe49364b2a1643d23f9699f95654735dc
Author: Sneha Vijayarajan
AuthorDate: Mon Aug 17 22:47:18 2020 +0530
Upgrade store REST API version to 2019-12-12
- Contributed by Sneha Vijayarajan
---
.../main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index e1ea75e..3ee2d7e 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -62,7 +62,7 @@ public class AbfsClient implements Closeable {
public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class);
private final URL baseUrl;
private final SharedKeyCredentials sharedKeyCredentials;
- private final String xMsVersion = "2018-11-09";
+ private final String xMsVersion = "2019-12-12";
private final ExponentialRetryPolicy retryPolicy;
private final String filesystem;
private final AbfsConfiguration abfsConfiguration;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17183. ABFS: Enabling checkaccess on ABFS
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a2610e2 HADOOP-17183. ABFS: Enabling checkaccess on ABFS
a2610e2 is described below
commit a2610e21ed5289323d8a6f6359477a8ceb2db2eb
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Fri Aug 7 03:22:02 2020 +0530
HADOOP-17183. ABFS: Enabling checkaccess on ABFS
- Contributed by Bilahari T H
---
.../constants/FileSystemConfigurations.java| 2 +-
.../ITestAzureBlobFileSystemCheckAccess.java | 110 +
2 files changed, 72 insertions(+), 40 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 260c496..f70d102 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -84,7 +84,7 @@ public final class FileSystemConfigurations {
public static final boolean DEFAULT_ENABLE_HTTPS = true;
public static final boolean DEFAULT_USE_UPN = false;
- public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = false;
+ public static final boolean DEFAULT_ENABLE_CHECK_ACCESS = true;
public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false;
public static final long
DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS = 120;
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
index 4189d66..45a3cbd 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
@@ -17,16 +17,18 @@
*/
package org.apache.hadoop.fs.azurebfs;
-import com.google.common.collect.Lists;
-
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.List;
+import com.google.common.collect.Lists;
import org.junit.Assume;
import org.junit.Test;
+import org.mockito.Mockito;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers;
@@ -37,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.AccessControlException;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET;
@@ -44,9 +47,19 @@ import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_A
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_ID;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CLIENT_SECRET;
import static
org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test cases for AzureBlobFileSystem.access()
+ *
+ * Some of the tests in this class requires the following 3 configs set in the
+ * test config file.
+ * fs.azure.account.test.oauth2.client.id
+ * fs.azure.account.test.oauth2.client.secret
+ * fs.azure.check.access.testuser.guid
+ * Set the above client id, secret and guid of a service principal which has no
+ * RBAC on the account.
+ *
*/
public class ITestAzureBlobFileSystemCheckAccess
extends AbstractAbfsIntegrationTest {
@@ -66,31 +79,29 @@ public class ITestAzureBlobFileSystemCheckAccess
this.isCheckAccessEnabled = getConfiguration().isCheckAccessEnabled();
this.isHNSEnabled = getConfiguration()
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
+setTestUserFs();
}
private void setTestUserFs() throws Exception {
if (this.testUserFs != null) {
return;
}
-
[hadoop] branch trunk updated (c566cab -> 3f73fac)
This is an automated email from the ASF dual-hosted git repository. dazhou pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from c566cab HADOOP-17163. ABFS: Adding debug log for rename failures add 3f73fac HADOOP-17149. ABFS: Fixing the testcase ITestGetNameSpaceEnabled No new revisions were added by this update. Summary of changes: .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 23 .../hadoop/fs/azurebfs/ITestSharedKeyAuth.java | 61 ++ 2 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17163. ABFS: Adding debug log for rename failures
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c566cab HADOOP-17163. ABFS: Adding debug log for rename failures
c566cab is described below
commit c566cabd62b57c982a47595c3699649bcdf188ae
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Wed Aug 5 22:08:13 2020 +0530
HADOOP-17163. ABFS: Adding debug log for rename failures
- Contributed by Bilahari T H
---
.../src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 84d6068..513150a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -323,6 +323,7 @@ public class AzureBlobFileSystem extends FileSystem {
abfsStore.rename(qualifiedSrcPath, qualifiedDstPath);
return true;
} catch(AzureBlobFileSystemException ex) {
+ LOG.debug("Rename operation failed. ", ex);
checkException(
src,
ex,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17137. ABFS: Makes the test cases in ITestAbfsNetworkStatistics agnostic
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a7fda2e HADOOP-17137. ABFS: Makes the test cases in
ITestAbfsNetworkStatistics agnostic
a7fda2e is described below
commit a7fda2e38f2a06e18c2929dff0be978d5e0ef9d5
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Sat Aug 1 00:57:57 2020 +0530
HADOOP-17137. ABFS: Makes the test cases in ITestAbfsNetworkStatistics
agnostic
- Contributed by Bilahari T H
---
.../fs/azurebfs/ITestAbfsNetworkStatistics.java| 63 +-
1 file changed, 38 insertions(+), 25 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
index b2e1301..9d76fb0 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsNetworkStatistics.java
@@ -32,6 +32,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CONNECTIONS_MADE;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.SEND_REQUESTS;
+
public class ITestAbfsNetworkStatistics extends AbstractAbfsIntegrationTest {
private static final Logger LOG =
@@ -56,6 +59,11 @@ public class ITestAbfsNetworkStatistics extends
AbstractAbfsIntegrationTest {
String testNetworkStatsString = "http_send";
long connectionsMade, requestsSent, bytesSent;
+metricMap = fs.getInstrumentationMap();
+long connectionsMadeBeforeTest = metricMap
+.get(CONNECTIONS_MADE.getStatName());
+long requestsMadeBeforeTest = metricMap.get(SEND_REQUESTS.getStatName());
+
/*
* Creating AbfsOutputStream will result in 1 connection made and 1 send
* request.
@@ -75,27 +83,26 @@ public class ITestAbfsNetworkStatistics extends
AbstractAbfsIntegrationTest {
/*
* Testing the network stats with 1 write operation.
*
- * connections_made : 3(getFileSystem()) + 1(AbfsOutputStream) +
2(flush).
+ * connections_made : (connections made above) + 2(flush).
*
- * send_requests : 1(getFileSystem()) + 1(AbfsOutputStream) + 2(flush).
+ * send_requests : (requests sent above) + 2(flush).
*
* bytes_sent : bytes wrote in AbfsOutputStream.
*/
- if
(fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(sendRequestPath).toString()))
{
+ long extraCalls = 0;
+ if (!fs.getAbfsStore()
+ .isAppendBlobKey(fs.makeQualified(sendRequestPath).toString())) {
// no network calls are made for hflush in case of appendblob
-connectionsMade = assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE,
-5, metricMap);
-requestsSent = assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, 3,
-metricMap);
- } else {
-connectionsMade = assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE,
-6, metricMap);
-requestsSent = assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS, 4,
-metricMap);
+extraCalls++;
}
+ long expectedConnectionsMade = connectionsMadeBeforeTest + extraCalls +
2;
+ long expectedRequestsSent = requestsMadeBeforeTest + extraCalls + 2;
+ connectionsMade = assertAbfsStatistics(CONNECTIONS_MADE,
+ expectedConnectionsMade, metricMap);
+ requestsSent = assertAbfsStatistics(SEND_REQUESTS, expectedRequestsSent,
+ metricMap);
bytesSent = assertAbfsStatistics(AbfsStatistic.BYTES_SENT,
testNetworkStatsString.getBytes().length, metricMap);
-
}
// To close the AbfsOutputStream 1 connection is made and 1 request is
sent.
@@ -135,14 +142,14 @@ public class ITestAbfsNetworkStatistics extends
AbstractAbfsIntegrationTest {
*/
if
(fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(sendRequestPath).toString()))
{
// no network calls are made for hflush in case of appendblob
-assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE,
+assertAbfsStatistics(CONNECTIONS_MADE,
connectionsMade + 1 + LARGE_OPERATIONS, metricMap);
-assertAbfsStatistics(AbfsStatistic.SEND_REQUESTS,
+assertAbfsStatistics(SEND_REQUESTS,
requestsSent + 1 + LARGE_OPERATIONS, metricMap);
} else {
-assertAbfsStatistics(AbfsStatistic.CONNECTIONS_MADE,
+assertAbfsStatistics(CONNECTIONS_MADE,
connectionsMade + 1 + LARGE_OPERATION
[hadoop] branch trunk updated: Hadoop 17132. ABFS: Fix Rename and Delete Idempotency check trigger
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d23cc9d Hadoop 17132. ABFS: Fix Rename and Delete Idempotency check
trigger
d23cc9d is described below
commit d23cc9d85d887f01d72180bdf1af87dfdee15c5a
Author: Sneha Vijayarajan
AuthorDate: Tue Jul 21 21:52:38 2020 +0530
Hadoop 17132. ABFS: Fix Rename and Delete Idempotency check trigger
- Contributed by Sneha Vijayarajan
---
.../hadoop/fs/azurebfs/services/AbfsClient.java| 55 +++-
.../fs/azurebfs/services/AbfsRestOperation.java| 7 +-
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 65 +-
.../azurebfs/ITestAzureBlobFileSystemRename.java | 68 +++
.../fs/azurebfs/services/TestAbfsClient.java | 76 ++
5 files changed, 253 insertions(+), 18 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index f747bd0..e1ea75e 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -336,10 +336,19 @@ public class AbfsClient implements Closeable {
url,
requestHeaders);
Instant renameRequestStartTime = Instant.now();
-op.execute();
-
-if (op.getResult().getStatusCode() != HttpURLConnection.HTTP_OK) {
- return renameIdempotencyCheckOp(renameRequestStartTime, op, destination);
+try {
+ op.execute();
+} catch (AzureBlobFileSystemException e) {
+final AbfsRestOperation idempotencyOp = renameIdempotencyCheckOp(
+renameRequestStartTime, op, destination);
+if (idempotencyOp.getResult().getStatusCode()
+== op.getResult().getStatusCode()) {
+ // idempotency did not return different result
+ // throw back the exception
+ throw e;
+} else {
+ return idempotencyOp;
+}
}
return op;
@@ -369,14 +378,21 @@ public class AbfsClient implements Closeable {
// exists. Check on destination status and if it has a recent LMT
timestamp.
// If yes, return success, else fall back to original rename request
failure response.
- final AbfsRestOperation destStatusOp = getPathStatus(destination, false);
- if (destStatusOp.getResult().getStatusCode() ==
HttpURLConnection.HTTP_OK) {
-String lmt = destStatusOp.getResult().getResponseHeader(
-HttpHeaderConfigurations.LAST_MODIFIED);
-
-if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) {
- return destStatusOp;
+ try {
+final AbfsRestOperation destStatusOp = getPathStatus(destination,
+false);
+if (destStatusOp.getResult().getStatusCode()
+== HttpURLConnection.HTTP_OK) {
+ String lmt = destStatusOp.getResult().getResponseHeader(
+ HttpHeaderConfigurations.LAST_MODIFIED);
+
+ if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) {
+return destStatusOp;
+ }
}
+ } catch (AzureBlobFileSystemException e) {
+// GetFileStatus on the destination failed, return original op
+return op;
}
}
@@ -570,10 +586,18 @@ public class AbfsClient implements Closeable {
HTTP_METHOD_DELETE,
url,
requestHeaders);
+try {
op.execute();
-
-if (op.getResult().getStatusCode() != HttpURLConnection.HTTP_OK) {
- return deleteIdempotencyCheckOp(op);
+} catch (AzureBlobFileSystemException e) {
+ final AbfsRestOperation idempotencyOp = deleteIdempotencyCheckOp(op);
+ if (idempotencyOp.getResult().getStatusCode()
+ == op.getResult().getStatusCode()) {
+// idempotency did not return different result
+// throw back the exception
+throw e;
+ } else {
+return idempotencyOp;
+ }
}
return op;
@@ -822,7 +846,8 @@ public class AbfsClient implements Closeable {
return createRequestUrl(EMPTY_STRING, query);
}
- private URL createRequestUrl(final String path, final String query)
+ @VisibleForTesting
+ protected URL createRequestUrl(final String path, final String query)
throws AzureBlobFileSystemException {
final String base = baseUrl.toString();
String encodedPath = path;
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index f3986d4..936267a 100644
---
a
[hadoop] branch trunk updated: HADOOP-17092. ABFS: Making AzureADAuthenticator.getToken() throw HttpException
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b4b23ef HADOOP-17092. ABFS: Making AzureADAuthenticator.getToken()
throw HttpException
b4b23ef is described below
commit b4b23ef0d1a0afe6251370a61f922ecdb1624165
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Tue Jul 21 21:48:54 2020 +0530
HADOOP-17092. ABFS: Making AzureADAuthenticator.getToken() throw
HttpException
- Contributed by Bilahari T H
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 27 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 5 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 6 ++
.../constants/FileSystemConfigurations.java| 8 ++
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 42 ++--
.../azurebfs/services/ExponentialRetryPolicy.java | 23
.../hadoop-azure/src/site/markdown/abfs.md | 20 +++-
.../services/TestAzureADAuthenticator.java | 120 +
8 files changed, 244 insertions(+), 7 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 4cdb3c9..85bd37a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -57,6 +57,7 @@ import
org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider;
import org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider;
import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.KeyProvider;
import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
@@ -119,6 +120,26 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT)
private int customTokenFetchRetryCount;
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT,
+ MinValue = 0,
+ DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS)
+ private int oauthTokenFetchRetryCount;
+
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF,
+ MinValue = 0,
+ DefaultValue =
DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL)
+ private int oauthTokenFetchRetryMinBackoff;
+
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF,
+ MinValue = 0,
+ DefaultValue =
DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL)
+ private int oauthTokenFetchRetryMaxBackoff;
+
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF,
+ MinValue = 0,
+ DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF)
+ private int oauthTokenFetchRetryDeltaBackoff;
+
@LongConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_BLOCK_SIZE_PROPERTY_NAME,
MinValue = 0,
MaxValue = MAX_AZURE_BLOCK_SIZE,
@@ -795,6 +816,12 @@ public class AbfsConfiguration{
validator.ThrowIfInvalid()).validate(value);
}
+ public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() {
+return new ExponentialRetryPolicy(oauthTokenFetchRetryCount,
+oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff,
+oauthTokenFetchRetryDeltaBackoff);
+ }
+
@VisibleForTesting
void setReadBufferSize(int bufferSize) {
this.readBufferSize = bufferSize;
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 66130d3..59c2e26 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -79,6 +79,7 @@ import org.apache.hadoop.fs.azurebfs.enums.Trilean;
import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider;
import org.apache.hadoop.fs.azurebfs.extensions.ExtensionHelper;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
+import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator;
import org.apache.hadoop.fs.azurebfs.oauth2.IdentityTransformer;
import org.apache.hadoop.fs.azurebfs.oauth2.IdentityTransformerInterface;
[hadoop] branch trunk updated: HADOOP-16682. ABFS: Removing unnecessary toString() invocations
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 99655167 HADOOP-16682. ABFS: Removing unnecessary toString()
invocations
99655167 is described below
commit 99655167f308b9c59e66b1b5d0d1fd5741cd75de
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Sat Jul 18 22:30:18 2020 +0530
HADOOP-16682. ABFS: Removing unnecessary toString() invocations
- Contributed by Bilahari T H
---
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 8
.../fs/azurebfs/AzureBlobFileSystemStore.java | 22 +++---
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index daa1905..84d6068 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -272,9 +272,9 @@ public class AzureBlobFileSystem extends FileSystem {
}
public boolean rename(final Path src, final Path dst) throws IOException {
-LOG.debug(
-"AzureBlobFileSystem.rename src: {} dst: {}", src.toString(),
dst.toString());
+LOG.debug("AzureBlobFileSystem.rename src: {} dst: {}", src, dst);
statIncrement(CALL_RENAME);
+
trailingPeriodCheck(dst);
Path parentFolder = src.getParent();
@@ -778,7 +778,7 @@ public class AzureBlobFileSystem extends FileSystem {
@Override
public void modifyAclEntries(final Path path, final List aclSpec)
throws IOException {
-LOG.debug("AzureBlobFileSystem.modifyAclEntries path: {}",
path.toString());
+LOG.debug("AzureBlobFileSystem.modifyAclEntries path: {}", path);
if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
@@ -927,7 +927,7 @@ public class AzureBlobFileSystem extends FileSystem {
*/
@Override
public AclStatus getAclStatus(final Path path) throws IOException {
-LOG.debug("AzureBlobFileSystem.getAclStatus path: {}", path.toString());
+LOG.debug("AzureBlobFileSystem.getAclStatus path: {}", path);
if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 74908de..66130d3 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -449,8 +449,8 @@ public class AzureBlobFileSystemStore implements Closeable {
client.getFileSystem(),
path,
overwrite,
- permission.toString(),
- umask.toString(),
+ permission,
+ umask,
isNamespaceEnabled);
String relativePath = getRelativePath(path);
@@ -903,7 +903,7 @@ public class AzureBlobFileSystemStore implements Closeable {
LOG.debug(
"setOwner filesystem: {} path: {} owner: {} group: {}",
client.getFileSystem(),
- path.toString(),
+ path,
owner,
group);
@@ -930,8 +930,8 @@ public class AzureBlobFileSystemStore implements Closeable {
LOG.debug(
"setPermission filesystem: {} path: {} permission: {}",
client.getFileSystem(),
- path.toString(),
- permission.toString());
+ path,
+ permission);
final AbfsRestOperation op = client.setPermission(getRelativePath(path),
String.format(AbfsHttpConstants.PERMISSION_FORMAT,
permission.toOctal()));
@@ -952,7 +952,7 @@ public class AzureBlobFileSystemStore implements Closeable {
LOG.debug(
"modifyAclEntries filesystem: {} path: {} aclSpec: {}",
client.getFileSystem(),
- path.toString(),
+ path,
AclEntry.aclSpecToString(aclSpec));
identityTransformer.transformAclEntriesForSetRequest(aclSpec);
@@ -993,7 +993,7 @@ public class AzureBlobFileSystemStore implements Closeable {
LOG.debug(
"removeAclEntries filesystem: {} path: {} aclSpec: {}",
client.getFileSystem(),
- path.toString(),
+ path,
AclEntry.aclSpecToString(aclSpec));
[hadoop] branch trunk updated: HADOOP-17058. ABFS: Support for AppendBlob in Hadoop ABFS Driver
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d20109c HADOOP-17058. ABFS: Support for AppendBlob in Hadoop ABFS
Driver
d20109c is described below
commit d20109c171460f3312a760c1309f95b2bf61e0d3
Author: ishaniahuja <50942176+ishaniah...@users.noreply.github.com>
AuthorDate: Sun Jul 5 01:55:14 2020 +0530
HADOOP-17058. ABFS: Support for AppendBlob in Hadoop ABFS Driver
- Contributed by Ishani Ahuja
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 50 ++-
.../fs/azurebfs/constants/AbfsHttpConstants.java | 1 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 3 +
.../constants/FileSystemConfigurations.java| 2 +
.../fs/azurebfs/constants/HttpQueryParams.java | 1 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 46 ++-
.../fs/azurebfs/services/AbfsHttpOperation.java| 10 +
.../fs/azurebfs/services/AbfsOutputStream.java | 63 ++-
.../azurebfs/services/AbfsOutputStreamContext.java | 12 +
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 4 +
.../fs/azurebfs/ITestAbfsNetworkStatistics.java| 52 ++-
.../azurebfs/ITestAbfsOutputStreamStatistics.java | 4 +
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 3 +
.../fs/azurebfs/ITestAbfsStreamStatistics.java | 11 +-
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 20 +-
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 3 +
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 30 +-
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../azurebfs/constants/TestConfigurationKeys.java | 1 +
.../fs/azurebfs/services/TestAbfsOutputStream.java | 430 +
21 files changed, 714 insertions(+), 42 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index bbe2274..43021c0 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -153,6 +153,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES)
private String azureAtomicDirs;
+ @StringConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_APPEND_BLOB_KEY,
+ DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES)
+ private String azureAppendBlobDirs;
+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
DefaultValue =
DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION)
private boolean createRemoteFileSystemDuringInitialization;
@@ -544,6 +548,10 @@ public class AbfsConfiguration{
return this.azureAtomicDirs;
}
+ public String getAppendBlobDirs() {
+return this.azureAppendBlobDirs;
+ }
+
public boolean getCreateRemoteFileSystemDuringInitialization() {
// we do not support creating the filesystem when AuthType is SAS
return this.createRemoteFileSystemDuringInitialization
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index c310e29..74908de 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
@@ -145,6 +146,11 @@ public class AzureBlobFileSystemStore implements Closeable
{
private final IdentityTransformerInterface identityTransformer;
private final AbfsPerfTracker abfsPerfTracker;
+ /**
+ * The set of directories where we should store files as append blobs.
+ */
+ private Set appendBlobDirSet;
+
public AzureBlobFileSystemStore(URI uri, boolean isSecureScheme,
Configuration configuration,
AbfsCounters abfsCounters) throws
IOException {
@@ -196,6 +202,23 @@ public
[hadoop] branch trunk updated: HADOOP-17053. ABFS: Fix Account-specific OAuth config setting parsing
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4c5cd75 HADOOP-17053. ABFS: Fix Account-specific OAuth config setting
parsing
4c5cd75 is described below
commit 4c5cd751e3911e350c7437dcb28c0ed67735f635
Author: Sneha Vijayarajan
AuthorDate: Wed May 27 13:56:09 2020 -0700
HADOOP-17053. ABFS: Fix Account-specific OAuth config setting parsing
Contributed by Sneha Vijayarajan
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 91 --
.../fs/azurebfs/TestAccountConfiguration.java | 193 -
2 files changed, 265 insertions(+), 19 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index b26bf53..bbe2274 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -325,25 +325,73 @@ public class AbfsConfiguration{
}
/**
- * Returns the account-specific Class if it exists, then looks for an
- * account-agnostic value, and finally tries the default value.
+ * Returns account-specific token provider class if it exists, else checks if
+ * an account-agnostic setting is present for token provider class if
AuthType
+ * matches with authType passed.
+ * @param authType AuthType effective on the account
* @param name Account-agnostic configuration key
* @param defaultValue Class returned if none is configured
* @param xface Interface shared by all possible values
+ * @param Interface class type
* @return Highest-precedence Class object that was found
*/
- public Class getClass(String name, Class
defaultValue, Class xface) {
+ public Class getTokenProviderClass(AuthType authType,
+ String name,
+ Class defaultValue,
+ Class xface) {
+Class tokenProviderClass = getAccountSpecificClass(name, defaultValue,
+xface);
+
+// If there is none set specific for account
+// fall back to generic setting if Auth Type matches
+if ((tokenProviderClass == null)
+&& (authType == getAccountAgnosticEnum(
+FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) {
+ tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface);
+}
+
+return (tokenProviderClass == null)
+? null
+: tokenProviderClass.asSubclass(xface);
+ }
+
+ /**
+ * Returns the account-specific class if it exists, else returns default
value.
+ * @param name Account-agnostic configuration key
+ * @param defaultValue Class returned if none is configured
+ * @param xface Interface shared by all possible values
+ * @param Interface class type
+ * @return Account specific Class object that was found
+ */
+ public Class getAccountSpecificClass(String name,
+ Class defaultValue,
+ Class xface) {
return rawConfig.getClass(accountConf(name),
-rawConfig.getClass(name, defaultValue, xface),
+defaultValue,
xface);
}
/**
- * Returns the account-specific password in string form if it exists, then
+ * Returns account-agnostic Class if it exists, else returns the default
value.
+ * @param name Account-agnostic configuration key
+ * @param defaultValue Class returned if none is configured
+ * @param xface Interface shared by all possible values
+ * @param Interface class type
+ * @return Account-Agnostic Class object that was found
+ */
+ public Class getAccountAgnosticClass(String name,
+ Class defaultValue,
+ Class xface) {
+return rawConfig.getClass(name, defaultValue, xface);
+ }
+
+ /**
+ * Returns the account-specific enum value if it exists, then
* looks for an account-agnostic value.
* @param name Account-agnostic configuration key
* @param defaultValue Value returned if none is configured
- * @return value in String form if one exists, else null
+ * @param Enum type
+ * @return enum value if one exists, else null
*/
public > T getEnum(String name, T defaultValue) {
return rawConfig.getEnum(accountConf(name),
@@ -351,6 +399,18 @@ public class AbfsConfiguration{
}
/**
+ * Returns the account-agnostic enum value if it exists, else
+ * return default.
+ * @param name Account-agnostic configuration key
+ * @param defaultValue Value returned if none is configured
+ * @param Enum type
+ * @return enum value if one exists, else null
+ */
+ public > T getAccountAgnosticEnum(String name, T
defaultValue) {
+return rawConfig.getEnum(name, defaultValue);
+ }
+
+ /**
* Unsets parameter in the underlying
[hadoop] branch trunk updated: HADOOP-16852: Report read-ahead error back
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 53b993e HADOOP-16852: Report read-ahead error back
53b993e is described below
commit 53b993e6048ffaaf98e460690211fc08efb20cf2
Author: Sneha Vijayarajan
AuthorDate: Wed May 27 13:51:42 2020 -0700
HADOOP-16852: Report read-ahead error back
Contributed by Sneha Vijayarajan
---
.../fs/azurebfs/services/AbfsInputStream.java | 12 +
.../hadoop/fs/azurebfs/services/ReadBuffer.java| 16 +
.../fs/azurebfs/services/ReadBufferManager.java| 90 -
.../fs/azurebfs/services/ReadBufferWorker.java | 12 +-
.../fs/azurebfs/services/TestAbfsInputStream.java | 450 +
.../fs/azurebfs/utils/TestCachedSASToken.java | 34 ++
6 files changed, 604 insertions(+), 10 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index 422fa3b..50380c9 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -24,6 +24,10 @@ import java.io.IOException;
import java.net.HttpURLConnection;
import com.google.common.base.Preconditions;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSExceptionMessages;
@@ -41,6 +45,7 @@ import static org.apache.hadoop.util.StringUtils.toLowerCase;
*/
public class AbfsInputStream extends FSInputStream implements CanUnbuffer,
StreamCapabilities {
+ private static final Logger LOG =
LoggerFactory.getLogger(AbfsInputStream.class);
private final AbfsClient client;
private final Statistics statistics;
@@ -239,6 +244,7 @@ public class AbfsInputStream extends FSInputStream
implements CanUnbuffer,
final AbfsRestOperation op;
AbfsPerfTracker tracker = client.getAbfsPerfTracker();
try (AbfsPerfInfo perfInfo = new AbfsPerfInfo(tracker, "readRemote",
"read")) {
+ LOG.trace("Trigger client.read for path={} position={} offset={}
length={}", path, position, offset, length);
op = client.read(path, position, b, offset, length, tolerateOobAppends ?
"*" : eTag, cachedSasToken.get());
cachedSasToken.update(op.getSasToken());
perfInfo.registerResult(op.getResult()).registerSuccess(true);
@@ -431,4 +437,10 @@ public class AbfsInputStream extends FSInputStream
implements CanUnbuffer,
byte[] getBuffer() {
return buffer;
}
+
+ @VisibleForTesting
+ protected void setCachedSasToken(final CachedSASToken cachedSasToken) {
+this.cachedSasToken = cachedSasToken;
+ }
+
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java
index 00e4f00..5d55726 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java
@@ -18,10 +18,13 @@
package org.apache.hadoop.fs.azurebfs.services;
+import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus;
+import static
org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus.READ_FAILED;
+
class ReadBuffer {
private AbfsInputStream stream;
@@ -40,6 +43,8 @@ class ReadBuffer {
private boolean isLastByteConsumed = false;
private boolean isAnyByteConsumed = false;
+ private IOException errException = null;
+
public AbfsInputStream getStream() {
return stream;
}
@@ -88,12 +93,23 @@ class ReadBuffer {
this.bufferindex = bufferindex;
}
+ public IOException getErrException() {
+return errException;
+ }
+
+ public void setErrException(final IOException errException) {
+this.errException = errException;
+ }
+
public ReadBufferStatus getStatus() {
return status;
}
public void setStatus(ReadBufferStatus status) {
this.status = status;
+if (status == READ_FAILED) {
+ bufferindex = -1;
+}
}
public CountDownLatch getLatch() {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
index 5b71cf0..73c23b0 100644
---
a/hadoop-tools/hadoop-azure/src/main/jav
[hadoop] branch trunk updated: HADOOP-17054. ABFS: Fix test AbfsClient authentication instance
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 37b1b47 HADOOP-17054. ABFS: Fix test AbfsClient authentication
instance
37b1b47 is described below
commit 37b1b4799db680d4a8bd4cb389e00d044f1e4a37
Author: Sneha Vijayarajan
AuthorDate: Tue May 26 15:26:28 2020 -0700
HADOOP-17054. ABFS: Fix test AbfsClient authentication instance
Contributed by Sneha Vijayarajan
---
.../fs/azurebfs/services/TestAbfsClient.java | 42 +-
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
index ce9c032..0fd65fb 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
@@ -246,21 +246,29 @@ public final class TestAbfsClient {
AbfsClient baseAbfsClientInstance,
AbfsConfiguration abfsConfig)
throws AzureBlobFileSystemException {
- AbfsPerfTracker tracker = new AbfsPerfTracker("test",
- abfsConfig.getAccountName(),
- abfsConfig);
-
- // Create test AbfsClient
- AbfsClient testClient = new AbfsClient(
- baseAbfsClientInstance.getBaseUrl(),
- new SharedKeyCredentials(abfsConfig.getAccountName().substring(0,
- abfsConfig.getAccountName().indexOf(DOT)),
- abfsConfig.getStorageAccountKey()),
- abfsConfig,
- new ExponentialRetryPolicy(abfsConfig.getMaxIoRetries()),
- abfsConfig.getTokenProvider(),
- tracker);
-
- return testClient;
-}
+AuthType currentAuthType = abfsConfig.getAuthType(
+abfsConfig.getAccountName());
+
+AbfsPerfTracker tracker = new AbfsPerfTracker("test",
+abfsConfig.getAccountName(),
+abfsConfig);
+
+// Create test AbfsClient
+AbfsClient testClient = new AbfsClient(
+baseAbfsClientInstance.getBaseUrl(),
+(currentAuthType == AuthType.SharedKey
+? new SharedKeyCredentials(
+abfsConfig.getAccountName().substring(0,
+abfsConfig.getAccountName().indexOf(DOT)),
+abfsConfig.getStorageAccountKey())
+: null),
+abfsConfig,
+new ExponentialRetryPolicy(abfsConfig.getMaxIoRetries()),
+(currentAuthType == AuthType.OAuth
+? abfsConfig.getTokenProvider()
+: null),
+tracker);
+
+return testClient;
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17004. Fixing a formatting issue
This is an automated email from the ASF dual-hosted git repository. dazhou pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new d2f7133 HADOOP-17004. Fixing a formatting issue d2f7133 is described below commit d2f7133c6220ab886dc838f3ebc8d89f077c8acc Author: bilaharith <52483117+bilahar...@users.noreply.github.com> AuthorDate: Thu May 21 00:21:48 2020 +0530 HADOOP-17004. Fixing a formatting issue Contributed by Bilahari T H. --- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 474e6f8..a8ac083 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -763,7 +763,7 @@ The following configs are related to read and write operations. `fs.azure.io.retry.max.retries`: Sets the number of retries for IO operations. Currently this is used only for the server call retry logic. Used within AbfsClient class as part of the ExponentialRetryPolicy. The value should be ->= 0. +greater than or equal to 0. `fs.azure.write.request.size`: To set the write buffer size. Specify the value in bytes. The value should be between 16384 to 104857600 both inclusive (16 KB - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Hadoop-17015. ABFS: Handling Rename and Delete idempotency
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8f78aeb Hadoop-17015. ABFS: Handling Rename and Delete idempotency
8f78aeb is described below
commit 8f78aeb2500011e568929b585ed5b0987355f88d
Author: Sneha Vijayarajan
AuthorDate: Tue May 19 12:30:07 2020 -0700
Hadoop-17015. ABFS: Handling Rename and Delete idempotency
Contributed by Sneha Vijayarajan.
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 5 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 20 +---
.../constants/FileSystemConfigurations.java| 3 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 87
.../fs/azurebfs/services/AbfsHttpOperation.java| 13 +++
.../fs/azurebfs/services/AbfsRestOperation.java| 20 +++-
.../hadoop/fs/azurebfs/utils/DateTimeUtils.java| 71 +
.../hadoop-azure/src/site/markdown/abfs.md | 12 +++
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 53 ++
.../azurebfs/ITestAzureBlobFileSystemRename.java | 113 -
.../TestAbfsConfigurationFieldsValidation.java | 7 ++
.../fs/azurebfs/services/TestAbfsClient.java | 23 +
12 files changed, 409 insertions(+), 18 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 354176f..b26bf53 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -765,6 +765,11 @@ public class AbfsConfiguration{
}
@VisibleForTesting
+ void setMaxBackoffIntervalMilliseconds(int maxBackoffInterval) {
+this.maxBackoffInterval = maxBackoffInterval;
+ }
+
+ @VisibleForTesting
void setIsNamespaceEnabledAccount(String isNamespaceEnabledAccount) {
this.isNamespaceEnabledAccount = isNamespaceEnabledAccount;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 8e0e6c1..45c9d68 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -34,7 +34,6 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
-import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.ArrayList;
@@ -96,6 +95,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker;
import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import org.apache.hadoop.fs.azurebfs.utils.CRC64;
+import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils;
import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -128,7 +128,6 @@ public class AzureBlobFileSystemStore implements Closeable {
private URI uri;
private String userName;
private String primaryUserGroup;
- private static final String DATE_TIME_PATTERN = "E, dd MMM HH:mm:ss z";
private static final String TOKEN_DATE_PATTERN =
"-MM-dd'T'HH:mm:ss.SSS'Z'";
private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1";
private static final int GET_SET_AGGREGATE_COUNT = 2;
@@ -672,7 +671,7 @@ public class AzureBlobFileSystemStore implements Closeable {
resourceIsDir,
1,
blockSize,
- parseLastModifiedTime(lastModified),
+ DateTimeUtils.parseLastModifiedTime(lastModified),
path,
eTag);
}
@@ -748,7 +747,8 @@ public class AzureBlobFileSystemStore implements Closeable {
long contentLength = entry.contentLength() == null ? 0 :
entry.contentLength();
boolean isDirectory = entry.isDirectory() == null ? false :
entry.isDirectory();
if (entry.lastModified() != null && !entry.lastModified().isEmpty())
{
-lastModifiedMillis = parseLastModifiedTime(entry.lastModified());
+lastModifiedMillis = DateTimeUtils.parseLastModifiedTime(
+entry.lastModified());
}
Path entryPath = new Path(File.separator + entry.name());
@@ -1235,18 +1235,6 @@ public class AzureBlobFileSystemStore impleme
[hadoop] branch trunk updated: HADOOP-17004. ABFS: Improve the ABFS driver documentation
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bdbd59c HADOOP-17004. ABFS: Improve the ABFS driver documentation
bdbd59c is described below
commit bdbd59cfa0904860fc4ce7a2afef1e84f35b8b82
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Tue May 19 09:15:54 2020 +0530
HADOOP-17004. ABFS: Improve the ABFS driver documentation
Contributed by Bilahari T H.
---
.../hadoop-azure/src/site/markdown/abfs.md | 133 -
1 file changed, 130 insertions(+), 3 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index 89f52e7..93141f1 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -257,7 +257,8 @@ will have the URL
`abfs://contain...@abfswales1.dfs.core.windows.net/`
You can create a new container through the ABFS connector, by setting the
option
- `fs.azure.createRemoteFileSystemDuringInitialization` to `true`.
+ `fs.azure.createRemoteFileSystemDuringInitialization` to `true`. Though the
+ same is not supported when AuthType is SAS.
If the container does not exist, an attempt to list it with `hadoop fs -ls`
will fail
@@ -317,8 +318,13 @@ driven by them.
What can be changed is what secrets/credentials are used to authenticate the
caller.
-The authentication mechanism is set in `fs.azure.account.auth.type` (or the
account specific variant),
-and, for the various OAuth options `fs.azure.account.oauth.provider.type`
+The authentication mechanism is set in `fs.azure.account.auth.type` (or the
+account specific variant). The possible values are SharedKey, OAuth, Custom
+and SAS. For the various OAuth options use the config `fs.azure.account
+.oauth.provider.type`. Following are the implementations supported
+ClientCredsTokenProvider, UserPasswordTokenProvider, MsiTokenProvider and
+RefreshTokenBasedTokenProvider. An IllegalArgumentException is thrown if
+the specified provider type is not one of the supported.
All secrets can be stored in JCEKS files. These are encrypted and password
protected —use them or a compatible Hadoop Key Management Store wherever
@@ -350,6 +356,15 @@ the password, "key", retrieved from the XML/JCECKs
configuration files.
*Note*: The source of the account key can be changed through a custom key
provider;
one exists to execute a shell script to retrieve it.
+A custom key provider class can be provided with the config
+`fs.azure.account.keyprovider`. If a key provider class is specified the same
+will be used to get account key. Otherwise the Simple key provider will be used
+which will use the key specified for the config `fs.azure.account.key`.
+
+To retrieve using shell script, specify the path to the script for the config
+`fs.azure.shellkeyprovider.script`. ShellDecryptionKeyProvider class use the
+script specified to retrieve the key.
+
### OAuth 2.0 Client Credentials
OAuth 2.0 credentials of (client id, client secret, endpoint) are provided in
the configuration/JCEKS file.
@@ -466,6 +481,13 @@ With an existing Oauth 2.0 token, make a request of the
Active Directory endpoin
+ fs.azure.account.oauth2.refresh.endpoint
+
+
+ Refresh token endpoint
+
+
+
fs.azure.account.oauth2.client.id
@@ -507,6 +529,13 @@ The Azure Portal/CLI is used to create the service
identity.
+ fs.azure.account.oauth2.msi.endpoint
+
+
+ MSI endpoint
+
+
+
fs.azure.account.oauth2.client.id
@@ -542,6 +571,26 @@ and optionally
`org.apache.hadoop.fs.azurebfs.extensions.BoundDTExtension`.
The declared class also holds responsibility to implement retry logic while
fetching access tokens.
+### Delegation Token
Provider
+
+A delegation token provider supplies the ABFS connector with delegation tokens,
+helps renew and cancel the tokens by implementing the
+CustomDelegationTokenManager interface.
+
+```xml
+
+ fs.azure.enable.delegation.token
+ true
+ Make this true to use delegation token provider
+
+
+ fs.azure.delegation.token.provider.type
+
{fully-qualified-class-name-for-implementation-of-CustomDelegationTokenManager-interface}
+
+```
+In case delegation token is enabled, and the config `fs.azure.delegation.token
+.provider.type` is not provided then an IlleagalArgumentException is thrown.
+
### Shared Access Signature (SAS) Token Provider
A Shared Access Signature (SAS) token provider supplies the ABFS connector
with SAS
@@ -691,6 +740,84 @@ Config `fs.azure.account.hns.enabled` provides an option
to specify whether
Config `fs.azure.enable.check.access` needs to be set true to enable
the AzureBlobFileSystem.access().
+### Primary User Group Options
+The group name
[hadoop] branch trunk updated (9827ff2 -> 30ef8d0)
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 9827ff2 YARN-10223. Remove jersey-test-framework-core dependency from
yarn-server-common. (#1939)
add 30ef8d0 HADOOP-17002. ABFS: Adding config to determine if the account
is HNS enabled or not
No new revisions were added by this update.
Summary of changes:
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 ++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 59 ++---
.../fs/azurebfs/constants/ConfigurationKeys.java | 7 +
.../constants/FileSystemConfigurations.java| 3 +
...eption.java => TrileanConversionException.java} | 19 +--
.../apache/hadoop/fs/azurebfs/enums/Trilean.java | 80
.../hadoop/fs/azurebfs/enums}/package-info.java| 2 +-
.../hadoop-azure/src/site/markdown/abfs.md | 5 +
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 141 -
.../apache/hadoop/fs/azurebfs/TrileanTests.java| 92 ++
10 files changed, 392 insertions(+), 30 deletions(-)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/{InvalidUriException.java
=> TrileanConversionException.java} (70%)
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/Trilean.java
copy
{hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token
=>
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums}/package-info.java
(95%)
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TrileanTests.java
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Hadoop 16857. ABFS: Stop CustomTokenProvider retry logic to depend on AbfsRestOp retry policy
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3d69383 Hadoop 16857. ABFS: Stop CustomTokenProvider retry logic to
depend on AbfsRestOp retry policy
3d69383 is described below
commit 3d69383c26649e272ce591061c919b8c96ee7cfc
Author: Sneha Vijayarajan
AuthorDate: Tue Apr 21 21:39:48 2020 -0700
Hadoop 16857. ABFS: Stop CustomTokenProvider retry logic to depend on
AbfsRestOp retry policy
Contributed by Sneha Vijayarajan
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 16 +++-
.../fs/azurebfs/constants/ConfigurationKeys.java | 1 +
.../constants/FileSystemConfigurations.java| 1 +
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 21 --
.../oauth2/CustomTokenProviderAdapter.java | 46 ++-
.../hadoop-azure/src/site/markdown/abfs.md | 2 +
.../azurebfs/ITestAbfsRestOperationException.java | 13 ++--
.../services/TestExponentialRetryPolicy.java | 88 ++
8 files changed, 174 insertions(+), 14 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 084b558..78d6260 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -110,6 +110,11 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS)
private int maxIoRetries;
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT,
+ MinValue = 0,
+ DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT)
+ private int customTokenFetchRetryCount;
+
@LongConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_BLOCK_SIZE_PROPERTY_NAME,
MinValue = 0,
MaxValue = MAX_AZURE_BLOCK_SIZE,
@@ -425,6 +430,10 @@ public class AbfsConfiguration{
return this.maxIoRetries;
}
+ public int getCustomTokenFetchRetryCount() {
+return this.customTokenFetchRetryCount;
+ }
+
public long getAzureBlockSize() {
return this.azureBlockSize;
}
@@ -597,7 +606,7 @@ public class AbfsConfiguration{
LOG.trace("Initializing {}", customTokenProviderClass.getName());
azureTokenProvider.initialize(rawConfig, accountName);
LOG.trace("{} init complete", customTokenProviderClass.getName());
-return new CustomTokenProviderAdapter(azureTokenProvider);
+return new CustomTokenProviderAdapter(azureTokenProvider,
getCustomTokenFetchRetryCount());
} catch(IllegalArgumentException e) {
throw e;
} catch (Exception e) {
@@ -732,6 +741,11 @@ public class AbfsConfiguration{
this.listMaxResults = listMaxResults;
}
+ @VisibleForTesting
+ public void setMaxIoRetries(int maxIoRetries) {
+this.maxIoRetries = maxIoRetries;
+ }
+
private String getTrimmedPasswordString(String key, String defaultValue)
throws IOException {
String value = getPasswordString(key);
if (StringUtils.isBlank(value)) {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 3d8b987..5db111b 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -36,6 +36,7 @@ public final class ConfigurationKeys {
public static final String AZURE_MAX_BACKOFF_INTERVAL =
"fs.azure.io.retry.max.backoff.interval";
public static final String AZURE_BACKOFF_INTERVAL =
"fs.azure.io.retry.backoff.interval";
public static final String AZURE_MAX_IO_RETRIES =
"fs.azure.io.retry.max.retries";
+ public static final String AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT =
"fs.azure.custom.token.fetch.retry.count";
// Read and write buffer sizes defined by the user
public static final String AZURE_WRITE_BUFFER_SIZE =
"fs.azure.write.request.size";
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 5c27d84..3add0ef 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/consta
[hadoop] branch branch-2 updated: HADOOP-16933. Backport HADOOP-16890 and HADOOP-16825 to branch-2
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 018c722 HADOOP-16933. Backport HADOOP-16890 and HADOOP-16825 to
branch-2
018c722 is described below
commit 018c72280a60b203ed01d0469ada42026fb97a7f
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Tue Apr 7 08:40:14 2020 +0530
HADOOP-16933. Backport HADOOP-16890 and HADOOP-16825 to branch-2
Hadoop 16890. Change in expiry calculation for MSI token provider.
Contributed by Bilahari T H
(cherry picked from commit 0b931f36ec83dc72729a9e84a0d313f471061c64)
HADOOP-16825: ITestAzureBlobFileSystemCheckAccess failing. Contributed by
Bilahari T H
(cherry picked from commit 5944d28130925fe1452f545e96b5e44f064bc69e)
---
hadoop-project/pom.xml | 5 ++
hadoop-tools/hadoop-azure/pom.xml | 6 ++
.../fs/azurebfs/oauth2/AccessTokenProvider.java| 2 +-
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 57 +
.../fs/azurebfs/oauth2/MsiTokenProvider.java | 34
.../fs/azurebfs/ITestAbfsMsiTokenProvider.java | 93 ++
.../ITestAzureBlobFileSystemCheckAccess.java | 21 -
7 files changed, 199 insertions(+), 19 deletions(-)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7aaf826..1f6a7dd 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1314,6 +1314,11 @@
+
+org.hamcrest
+hamcrest-library
+1.3
+
diff --git a/hadoop-tools/hadoop-azure/pom.xml
b/hadoop-tools/hadoop-azure/pom.xml
index 245c883..f4fc358 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -249,6 +249,12 @@
mockito-all
test
+
+ org.hamcrest
+ hamcrest-library
+ test
+
+
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AccessTokenProvider.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AccessTokenProvider.java
index 72f37a1..a20e6df 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AccessTokenProvider.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AccessTokenProvider.java
@@ -72,7 +72,7 @@ public abstract class AccessTokenProvider {
*
* @return true if the token is expiring in next 5 minutes
*/
- private boolean isTokenAboutToExpire() {
+ protected boolean isTokenAboutToExpire() {
if (token == null) {
LOG.debug("AADToken: no token. Returning expiring=true");
return true; // no token should have same response as expired token
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
index 0eb379c..03be1c0 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
@@ -136,7 +136,7 @@ public final class AzureADAuthenticator {
headers.put("Metadata", "true");
LOG.debug("AADToken: starting to fetch token using MSI");
-return getTokenCall(authEndpoint, qp.serialize(), headers, "GET");
+return getTokenCall(authEndpoint, qp.serialize(), headers, "GET", true);
}
/**
@@ -197,8 +197,13 @@ public final class AzureADAuthenticator {
}
private static AzureADToken getTokenCall(String authEndpoint, String body,
- Hashtable headers,
String httpMethod)
- throws IOException {
+ Hashtable headers, String httpMethod) throws IOException
{
+return getTokenCall(authEndpoint, body, headers, httpMethod, false);
+ }
+
+ private static AzureADToken getTokenCall(String authEndpoint, String body,
+ Hashtable headers, String httpMethod, boolean isMsi)
+ throws IOException {
AzureADToken token = null;
ExponentialRetryPolicy retryPolicy
= new ExponentialRetryPolicy(3, 0, 1000, 2);
@@ -211,7 +216,7 @@ public final class AzureADAuthenticator {
httperror = 0;
ex = null;
try {
-token = getTokenSingleCall(authEndpoint, body, headers, httpMethod);
+token = getTokenSingleCall(authEndpoint, body, headers, httpMethod,
isMsi);
} catch (HttpException e) {
httperror = e.httpErrorCode;
ex = e;
@@ -227,8 +232,9 @@ public final class AzureADAuthenticator {
return token;
}
- private static Azure
[hadoop] branch trunk updated: HADOOP-16826. ABFS: update abfs.md to include config keys for identity transformation
This is an automated email from the ASF dual-hosted git repository. dazhou pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 978c487 HADOOP-16826. ABFS: update abfs.md to include config keys for identity transformation 978c487 is described below commit 978c487672edd9f18d8e2c9a1da063ae789bd774 Author: Karthick Narendran AuthorDate: Thu Jan 23 20:35:57 2020 -0800 HADOOP-16826. ABFS: update abfs.md to include config keys for identity transformation Contributed by Karthick Narendran --- .../hadoop-azure/src/site/markdown/abfs.md | 31 ++ 1 file changed, 31 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 1d01e02..79ec2ad 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -857,6 +857,37 @@ signon page for humans, even though it is a machine calling. 1. The URL is wrong —it is pointing at a web page unrelated to OAuth2.0 1. There's a proxy server in the way trying to return helpful instructions. +### `java.io.IOException: The ownership on the staging directory /tmp/hadoop-yarn/staging/user1/.staging is not as expected. It is owned by . The directory must be owned by the submitter user1 or user1` + +When using [Azure Managed Identities](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview), the files/directories in ADLS Gen2 by default will be owned by the service principal object id i.e. principal ID & submitting jobs as the local OS user 'user1' results in the above exception. + +The fix is to mimic the ownership to the local OS user, by adding the below properties to`core-site.xml`. + +```xml + + fs.azure.identity.transformer.service.principal.id + service principal object id + + An Azure Active Directory object ID (oid) used as the replacement for names contained + in the list specified by “fs.azure.identity.transformer.service.principal.substitution.list”. + Notice that instead of setting oid, you can also set $superuser here. + + + + fs.azure.identity.transformer.service.principal.substitution.list + user1 + + A comma separated list of names to be replaced with the service principal ID specified by + “fs.azure.identity.transformer.service.principal.id”. This substitution occurs + when setOwner, setAcl, modifyAclEntries, or removeAclEntries are invoked with identities + contained in the substitution list. Notice that when in non-secure cluster, asterisk symbol * + can be used to match all user/group. + + +``` + +Once the above properties are configured, `hdfs dfs -ls abfs://contain...@abfswales1.dfs.core.windows.net/` shows the ADLS Gen2 files/directories are now owned by 'user1'. + ## Testing ABFS See the relevant section in [Testing Azure](testing_azure.html). - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16005. NativeAzureFileSystem does not support setXAttr.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c36f09d HADOOP-16005. NativeAzureFileSystem does not support setXAttr.
c36f09d is described below
commit c36f09deb91454c086926c01f872d8ca4419aee0
Author: Clemens Wolff
AuthorDate: Tue Jan 14 17:28:37 2020 -0800
HADOOP-16005. NativeAzureFileSystem does not support setXAttr.
Contributed by Clemens Wolff.
---
.../fs/azure/AzureNativeFileSystemStore.java | 67 ++---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 71 ++
.../hadoop/fs/azure/NativeFileSystemStore.java | 4 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 79
.../fs/azurebfs/AzureBlobFileSystemStore.java | 9 ++
.../fs/azure/NativeAzureFileSystemBaseTest.java| 62
.../ITestAzureBlobFileSystemAttributes.java| 104 +
7 files changed, 384 insertions(+), 12 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 239dec2..414d2f2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -29,6 +29,8 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.util.Calendar;
import java.util.Date;
@@ -247,6 +249,8 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
private static final int DEFAULT_CONCURRENT_WRITES = 8;
+ private static final Charset METADATA_ENCODING = StandardCharsets.UTF_8;
+
// Concurrent reads reads of data written out of band are disable by default.
//
private static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false;
@@ -1662,17 +1666,30 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
removeMetadataAttribute(blob, OLD_IS_FOLDER_METADATA_KEY);
}
- private static void storeLinkAttribute(CloudBlobWrapper blob,
- String linkTarget) throws UnsupportedEncodingException {
-// We have to URL encode the link attribute as the link URI could
+ private static String encodeMetadataAttribute(String value) throws
UnsupportedEncodingException {
+// We have to URL encode the attribute as it could
// have URI special characters which unless encoded will result
// in 403 errors from the server. This is due to metadata properties
// being sent in the HTTP header of the request which is in turn used
// on the server side to authorize the request.
-String encodedLinkTarget = null;
-if (linkTarget != null) {
- encodedLinkTarget = URLEncoder.encode(linkTarget, "UTF-8");
-}
+return value == null ? null : URLEncoder.encode(value,
METADATA_ENCODING.name());
+ }
+
+ private static String decodeMetadataAttribute(String encoded) throws
UnsupportedEncodingException {
+return encoded == null ? null : URLDecoder.decode(encoded,
METADATA_ENCODING.name());
+ }
+
+ private static String ensureValidAttributeName(String attribute) {
+// Attribute names must be valid C# identifiers so we have to
+// convert the namespace dots (e.g. "user.something") in the
+// attribute names. Using underscores here to be consistent with
+// the constant metadata keys defined earlier in the file
+return attribute.replace('.', '_');
+ }
+
+ private static void storeLinkAttribute(CloudBlobWrapper blob,
+ String linkTarget) throws UnsupportedEncodingException {
+String encodedLinkTarget = encodeMetadataAttribute(linkTarget);
storeMetadataAttribute(blob,
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
encodedLinkTarget);
@@ -1686,11 +1703,7 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
String encodedLinkTarget = getMetadataAttribute(blob,
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
-String linkTarget = null;
-if (encodedLinkTarget != null) {
- linkTarget = URLDecoder.decode(encodedLinkTarget, "UTF-8");
-}
-return linkTarget;
+return decodeMetadataAttribute(encodedLinkTarget);
}
private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
@@ -2212,6 +2225,36 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
}
@Override
+ public byte[] retr
[hadoop] branch branch-2 updated: HADOOP-16778. ABFS: Backport HADOOP-16660 ABFS: Make RetryCount in ExponentialRetryPolicy Configurable to Branch-2. Contributed by Sneha Vijayarajan.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new ca00dcd HADOOP-16778. ABFS: Backport HADOOP-16660 ABFS: Make
RetryCount in ExponentialRetryPolicy Configurable to Branch-2. Contributed by
Sneha Vijayarajan.
ca00dcd is described below
commit ca00dcd26011c182c5152202c6ef776981b94597
Author: Sneha Vijayarajan
AuthorDate: Sat Jan 4 15:39:11 2020 -0800
HADOOP-16778. ABFS: Backport HADOOP-16660 ABFS: Make RetryCount in
ExponentialRetryPolicy Configurable to Branch-2.
Contributed by Sneha Vijayarajan.
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 4 +-
.../fs/azurebfs/services/AbfsRestOperation.java| 3 +
.../azurebfs/services/ExponentialRetryPolicy.java | 11 ++--
.../azurebfs/ITestAbfsRestOperationException.java | 43 +-
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 67 ++
6 files changed, 120 insertions(+), 10 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 158f834..cb12645 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -938,7 +938,9 @@ public class AzureBlobFileSystemStore {
tokenProvider = abfsConfiguration.getTokenProvider();
}
-this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, new
ExponentialRetryPolicy(), tokenProvider);
+this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
+new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()),
+tokenProvider);
}
private String getOctalNotation(FsPermission fsPermission) {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index fa8f742..54fe14a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -122,8 +122,11 @@ public class AbfsRestOperation {
*/
void execute() throws AzureBlobFileSystemException {
int retryCount = 0;
+LOG.debug("First execution of REST operation - {}", operationType);
while (!executeHttpOperation(retryCount++)) {
try {
+LOG.debug("Retrying REST operation {}. RetryCount = {}",
+operationType, retryCount);
Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
index 5eb7a66..b272cf2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
@@ -26,11 +26,6 @@ import java.net.HttpURLConnection;
* */
public class ExponentialRetryPolicy {
/**
- * Represents the default number of retry attempts.
- */
- private static final int DEFAULT_CLIENT_RETRY_COUNT = 30;
-
- /**
* Represents the default amount of time used when calculating a random
delta in the exponential
* delay between retries.
*/
@@ -86,8 +81,10 @@ public class ExponentialRetryPolicy {
/**
* Initializes a new instance of the {@link ExponentialRetryPolicy} class.
*/
- public ExponentialRetryPolicy() {
-this(DEFAULT_CLIENT_RETRY_COUNT, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
DEFAULT_CLIENT_BACKOFF);
+ public ExponentialRetryPolicy(final int maxIoRetries) {
+
+this(maxIoRetries, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
+DEFAULT_CLIENT_BACKOFF);
}
/**
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index ff88b02..cc66cca 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -20,
[hadoop] branch branch-2 updated: HADOOP-16734. Backport HADOOP-16455- "ABFS: Implement FileSystem.access() method" to branch-2. Contributed by Bilahari T H.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 48c564d HADOOP-16734. Backport HADOOP-16455- "ABFS: Implement
FileSystem.access() method" to branch-2. Contributed by Bilahari T H.
48c564d is described below
commit 48c564d5e3d906ff460ef729480fa6df4bf30a2e
Author: Bilahari T H
AuthorDate: Sat Jan 4 15:29:59 2020 -0800
HADOOP-16734. Backport HADOOP-16455- "ABFS: Implement FileSystem.access()
method" to branch-2.
Contributed by Bilahari T H.
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 23 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 19 ++
.../fs/azurebfs/constants/AbfsHttpConstants.java | 1 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 4 +
.../constants/FileSystemConfigurations.java| 4 +-
.../fs/azurebfs/constants/HttpQueryParams.java | 1 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 22 ++
.../azurebfs/services/AbfsRestOperationType.java | 3 +-
.../hadoop-azure/src/site/markdown/abfs.md | 4 +
.../ITestAzureBlobFileSystemCheckAccess.java | 320 +
.../azurebfs/constants/TestConfigurationKeys.java | 8 +
12 files changed, 412 insertions(+), 5 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index ffddc45..ccc5463 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -178,6 +178,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_USE_UPN)
private boolean useUpn;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
+ FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS)
+ private boolean isCheckAccessEnabled;
+
private Map storageAccountKeys;
public AbfsConfiguration(final Configuration rawConfig, String accountName)
@@ -399,6 +403,10 @@ public class AbfsConfiguration{
return this.azureBlockSize;
}
+ public boolean isCheckAccessEnabled() {
+return this.isCheckAccessEnabled;
+ }
+
public String getAzureBlockLocationHost() {
return this.azureBlockLocationHost;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 82a38a2..c3791fb 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -854,9 +854,14 @@ public class AzureBlobFileSystem extends FileSystem {
* @throws IOException see specific implementation
*/
@Override
- public void access(final Path path, FsAction mode) throws IOException {
-// TODO: make it no-op to unblock hive permission issue for now.
-// Will add a long term fix similar to the implementation in AdlFileSystem.
+ public void access(final Path path, final FsAction mode) throws IOException {
+LOG.debug("AzureBlobFileSystem.access path : {}, mode : {}", path, mode);
+Path qualifiedPath = makeQualified(path);
+try {
+ this.abfsStore.access(qualifiedPath, mode);
+} catch (AzureBlobFileSystemException ex) {
+ checkCheckAccessException(path, ex);
+}
}
private FileStatus tryGetFileStatus(final Path f) {
@@ -969,6 +974,18 @@ public class AzureBlobFileSystem extends FileSystem {
}
}
+ private void checkCheckAccessException(final Path path,
+ final AzureBlobFileSystemException exception) throws IOException {
+if (exception instanceof AbfsRestOperationException) {
+ AbfsRestOperationException ere = (AbfsRestOperationException) exception;
+ if (ere.getStatusCode() == HttpURLConnection.HTTP_FORBIDDEN) {
+throw (IOException) new AccessControlException(ere.getMessage())
+.initCause(exception);
+ }
+}
+checkException(path, exception);
+ }
+
/**
* Given a path and exception, choose which IOException subclass
* to create.
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 6710275..158f834 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java
[hadoop] 01/06: HADOOP-16548 : Disable Flush() over config
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 546db6428e83032c1fdbcc6625681c69cd6a76f6
Author: Sneha Vijayarajan
AuthorDate: Sat Sep 28 20:39:42 2019 -0700
HADOOP-16548 : Disable Flush() over config
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 +++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +++--
.../fs/azurebfs/constants/ConfigurationKeys.java | 8 +++
.../constants/FileSystemConfigurations.java| 1 +
.../fs/azurebfs/services/AbfsOutputStream.java | 7 --
.../hadoop-azure/src/site/markdown/abfs.md | 17 ++
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +++---
7 files changed, 63 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 67055c5..56ff622 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -146,6 +146,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_ENABLE_FLUSH)
private boolean enableFlush;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH,
+ DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH)
+ private boolean disableOutputStreamFlush;
+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ENABLE_AUTOTHROTTLING,
DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING)
private boolean enableAutoThrottling;
@@ -427,6 +431,10 @@ public class AbfsConfiguration{
return this.enableFlush;
}
+ public boolean isOutputStreamFlushDisabled() {
+return this.disableOutputStreamFlush;
+ }
+
public boolean isAutoThrottlingEnabled() {
return this.enableAutoThrottling;
}
@@ -635,4 +643,10 @@ public class AbfsConfiguration{
void setEnableFlush(boolean enableFlush) {
this.enableFlush = enableFlush;
}
+
+ @VisibleForTesting
+ void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) {
+this.disableOutputStreamFlush = disableOutputStreamFlush;
+ }
+
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index ce0d411..75fc08e 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -354,7 +354,8 @@ public class AzureBlobFileSystemStore {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
0,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void createDirectory(final Path path, final FsPermission permission,
final FsPermission umask)
@@ -426,7 +427,8 @@ public class AzureBlobFileSystemStore {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
offset,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void rename(final Path source, final Path destination) throws
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 8cd86bf..cd86f18 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -51,7 +51,15 @@ public final class ConfigurationKeys {
public static final String FS_AZURE_ALWAYS_USE_HTTPS =
"fs.azure.always.use.https";
public static final String FS_AZURE_ATOMIC_RENAME_KEY =
"fs.azure.atomic.rename.key";
public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH =
"fs.azure.readaheadqueue.depth";
+ /** Provides a config control to enable or disable ABFS Flush operations -
+ * HFlush and HSync. Default is true. **/
public static final String FS_AZURE_ENABLE_FLUSH = "fs.azure.enable.flush";
+ /** Provides a config control to disable or enable OutputStream Flush API
+ * operations in AbfsOutputStream. Flush() will trigger actions that
+ * guarantee that buffered data is p
[hadoop] 05/06: HADOOP-16455. ABFS: Implement FileSystem.access() method.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c225efe237937bda3c947d01f35d35eab0deda5f
Author: bilaharith <52483117+bilahar...@users.noreply.github.com>
AuthorDate: Wed Nov 27 21:26:38 2019 +0530
HADOOP-16455. ABFS: Implement FileSystem.access() method.
Contributed by Bilahari T H.
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 23 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 19 ++
.../fs/azurebfs/constants/AbfsHttpConstants.java | 1 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 4 +
.../constants/FileSystemConfigurations.java| 3 +-
.../fs/azurebfs/constants/HttpQueryParams.java | 1 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 22 ++
.../azurebfs/services/AbfsRestOperationType.java | 3 +-
.../hadoop-azure/src/site/markdown/abfs.md | 4 +
.../ITestAzureBlobFileSystemCheckAccess.java | 314 +
.../azurebfs/constants/TestConfigurationKeys.java | 8 +
12 files changed, 405 insertions(+), 5 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 052ae01..fde4fc7 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -178,6 +178,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_USE_UPN)
private boolean useUpn;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
+ FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS)
+ private boolean isCheckAccessEnabled;
+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ABFS_LATENCY_TRACK,
DefaultValue = DEFAULT_ABFS_LATENCY_TRACK)
private boolean trackLatency;
@@ -403,6 +407,10 @@ public class AbfsConfiguration{
return this.azureBlockSize;
}
+ public boolean isCheckAccessEnabled() {
+return this.isCheckAccessEnabled;
+ }
+
public String getAzureBlockLocationHost() {
return this.azureBlockLocationHost;
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 82a38a2..c3791fb 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -854,9 +854,14 @@ public class AzureBlobFileSystem extends FileSystem {
* @throws IOException see specific implementation
*/
@Override
- public void access(final Path path, FsAction mode) throws IOException {
-// TODO: make it no-op to unblock hive permission issue for now.
-// Will add a long term fix similar to the implementation in AdlFileSystem.
+ public void access(final Path path, final FsAction mode) throws IOException {
+LOG.debug("AzureBlobFileSystem.access path : {}, mode : {}", path, mode);
+Path qualifiedPath = makeQualified(path);
+try {
+ this.abfsStore.access(qualifiedPath, mode);
+} catch (AzureBlobFileSystemException ex) {
+ checkCheckAccessException(path, ex);
+}
}
private FileStatus tryGetFileStatus(final Path f) {
@@ -969,6 +974,18 @@ public class AzureBlobFileSystem extends FileSystem {
}
}
+ private void checkCheckAccessException(final Path path,
+ final AzureBlobFileSystemException exception) throws IOException {
+if (exception instanceof AbfsRestOperationException) {
+ AbfsRestOperationException ere = (AbfsRestOperationException) exception;
+ if (ere.getStatusCode() == HttpURLConnection.HTTP_FORBIDDEN) {
+throw (IOException) new AccessControlException(ere.getMessage())
+.initCause(exception);
+ }
+}
+checkException(path, exception);
+ }
+
/**
* Given a path and exception, choose which IOException subclass
* to create.
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 5acb597..27d7f92 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -1064,6 +1064,25 @@ public class AzureBlobFileSystemStore {
}
}
+ public void access(final Path path, fin
[hadoop] 06/06: HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy Configurable.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit aa9cd0a2d641bef91b3d9d4054d52e8b0ee61bc1
Author: Sneha Vijayarajan
AuthorDate: Wed Nov 27 15:07:27 2019 -0800
HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy Configurable.
Contributed by Sneha Vijayarajan.
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +-
.../fs/azurebfs/services/AbfsRestOperation.java| 3 +
.../azurebfs/services/ExponentialRetryPolicy.java | 11 ++--
.../azurebfs/ITestAbfsRestOperationException.java | 41 +
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 67 ++
6 files changed, 120 insertions(+), 10 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 27d7f92..3b4c219 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -1118,7 +1118,9 @@ public class AzureBlobFileSystemStore {
tokenProvider = abfsConfiguration.getTokenProvider();
}
-this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, new
ExponentialRetryPolicy(), tokenProvider, abfsPerfTracker);
+this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
+new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()),
+tokenProvider, abfsPerfTracker);
}
private String getOctalNotation(FsPermission fsPermission) {
@@ -1342,4 +1344,4 @@ public class AzureBlobFileSystemStore {
AbfsClient getClient() {
return this.client;
}
-}
+}
\ No newline at end of file
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 4803b6c..ddcb222 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -130,8 +130,11 @@ public class AbfsRestOperation {
}
int retryCount = 0;
+LOG.debug("First execution of REST operation - {}", operationType);
while (!executeHttpOperation(retryCount++)) {
try {
+LOG.debug("Retrying REST operation {}. RetryCount = {}",
+operationType, retryCount);
Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
index 5eb7a66..b272cf2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
@@ -26,11 +26,6 @@ import java.net.HttpURLConnection;
* */
public class ExponentialRetryPolicy {
/**
- * Represents the default number of retry attempts.
- */
- private static final int DEFAULT_CLIENT_RETRY_COUNT = 30;
-
- /**
* Represents the default amount of time used when calculating a random
delta in the exponential
* delay between retries.
*/
@@ -86,8 +81,10 @@ public class ExponentialRetryPolicy {
/**
* Initializes a new instance of the {@link ExponentialRetryPolicy} class.
*/
- public ExponentialRetryPolicy() {
-this(DEFAULT_CLIENT_RETRY_COUNT, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
DEFAULT_CLIENT_BACKOFF);
+ public ExponentialRetryPolicy(final int maxIoRetries) {
+
+this(maxIoRetries, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
+DEFAULT_CLIENT_BACKOFF);
}
/**
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index ff88b02..6f69826 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -20,12 +20,17 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoo
[hadoop] 03/06: HADOOP-16587. Make ABFS AAD endpoints configurable.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ffeb6d8ecee0c9d6b8cda355a28f35f2f8343cec
Author: bilaharith
AuthorDate: Mon Oct 7 13:07:32 2019 +0100
HADOOP-16587. Make ABFS AAD endpoints configurable.
Contributed by Bilahari T H.
This also addresses HADOOP-16498: AzureADAuthenticator cannot authenticate
in China.
Change-Id: I2441dd48b50b59b912b0242f7f5a4418cf94a87c
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 35 -
.../fs/azurebfs/constants/AuthConfigurations.java | 45 ++
.../fs/azurebfs/constants/ConfigurationKeys.java | 6 +++
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 23 ++-
.../fs/azurebfs/oauth2/MsiTokenProvider.java | 14 +--
.../oauth2/RefreshTokenBasedTokenProvider.java | 12 --
6 files changed, 118 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 56ff622..ffddc45 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -25,9 +25,12 @@ import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
+import org.apache.hadoop.fs.azurebfs.constants.AuthConfigurations;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@@ -69,6 +72,7 @@ import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class AbfsConfiguration{
+
private final Configuration rawConfig;
private final String accountName;
private final boolean isSecure;
@@ -486,13 +490,25 @@ public class AbfsConfiguration{
String password =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD);
tokenProvider = new UserPasswordTokenProvider(authEndpoint,
username, password);
} else if (tokenProviderClass == MsiTokenProvider.class) {
+ String authEndpoint = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
+ AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT);
String tenantGuid =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT);
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
- tokenProvider = new MsiTokenProvider(tenantGuid, clientId);
+ String authority = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY,
+ AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY);
+ authority = appendSlashIfNeeded(authority);
+ tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid,
+ clientId, authority);
} else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class)
{
+ String authEndpoint = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT,
+
AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT);
String refreshToken =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN);
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
- tokenProvider = new RefreshTokenBasedTokenProvider(clientId,
refreshToken);
+ tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint,
+ clientId, refreshToken);
} else {
throw new IllegalArgumentException("Failed to initialize " +
tokenProviderClass);
}
@@ -649,4 +665,19 @@ public class AbfsConfiguration{
this.disableOutputStreamFlush = disableOutputStreamFlush;
}
+ private String getTrimmedPasswordString(String key, String defaultValue)
throws IOException {
+String value = getPasswordString(key);
+if (StringUtils.isBlank(value)) {
+ value = defaultValue;
+}
+return value.trim();
+ }
+
+ private String appendSlashIfNeeded(String authority) {
+if (!authority.endsWith(AbfsHttpConstants.FOR
[hadoop] 04/06: HADOOP-16612. Track Azure Blob File System client-perceived latency
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b1e748f45b68855a37d53fabc480f1ebb8752606
Author: Jeetesh Mangwani
AuthorDate: Tue Nov 19 08:51:49 2019 -0800
HADOOP-16612. Track Azure Blob File System client-perceived latency
Contributed by Jeetesh Mangwani.
This add the ability to track the end-to-end performance of ADLS Gen 2 REST
APIs by measuring latency in the Hadoop ABFS driver.
The latency information is sent back to the ADLS Gen 2 REST API endpoints
in the subsequent requests.
---
hadoop-tools/hadoop-azure/pom.xml | 7 +-
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 13 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 844 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 2 +
.../constants/FileSystemConfigurations.java| 1 +
.../constants/HttpHeaderConfigurations.java| 1 +
.../contracts/services/AbfsPerfLoggable.java | 34 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 9 +-
.../fs/azurebfs/services/AbfsHttpOperation.java| 46 +-
.../fs/azurebfs/services/AbfsInputStream.java | 4 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 21 +-
.../hadoop/fs/azurebfs/services/AbfsPerfInfo.java | 133
.../fs/azurebfs/services/AbfsPerfTracker.java | 319
.../fs/azurebfs/services/AbfsRestOperation.java| 8 +
.../hadoop-azure/src/site/markdown/abfs.md | 46 ++
.../fs/azurebfs/services/TestAbfsClient.java | 2 +-
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 408 ++
.../hadoop-azure/src/test/resources/azure-test.xml | 5 +
18 files changed, 1562 insertions(+), 341 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml
b/hadoop-tools/hadoop-azure/pom.xml
index f079b71..c40b6bb 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -262,7 +262,12 @@
mockito-all
test
-
+
+
+ org.assertj
+ assertj-core
+ test
+
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index ffddc45..052ae01 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -178,6 +178,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_USE_UPN)
private boolean useUpn;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ABFS_LATENCY_TRACK,
+ DefaultValue = DEFAULT_ABFS_LATENCY_TRACK)
+ private boolean trackLatency;
+
private Map storageAccountKeys;
public AbfsConfiguration(final Configuration rawConfig, String accountName)
@@ -471,6 +475,15 @@ public class AbfsConfiguration{
return this.useUpn;
}
+ /**
+ * Whether {@code AbfsClient} should track and send latency info back to
storage servers.
+ *
+ * @return a boolean indicating whether latency should be tracked.
+ */
+ public boolean shouldTrackLatency() {
+return this.trackLatency;
+ }
+
public AccessTokenProvider getTokenProvider() throws
TokenAccessProviderException {
AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
AuthType.SharedKey);
if (authType == AuthType.OAuth) {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 75fc08e..5acb597 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -34,6 +34,7 @@ import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -80,6 +81,8 @@ import
org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
+import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker;
+import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import org.apache.hadoop.fs.azurebfs.utils.CRC64;
import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
@@ -119,6 +122,7 @@ public class AzureBlobFileSystemStore {
private static final String
[hadoop] branch branch-3.2 updated (26367b6 -> aa9cd0a)
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a change to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 26367b6 Bump nimbus-jose-jwt from 4.41.1 to 7.9 (#1682)
new 546db64 HADOOP-16548 : Disable Flush() over config
new 8b2c7e0 HADOOP-16578 : Avoid FileSystem API calls when FileSystem
already exists
new ffeb6d8 HADOOP-16587. Make ABFS AAD endpoints configurable.
new b1e748f HADOOP-16612. Track Azure Blob File System client-perceived
latency
new c225efe HADOOP-16455. ABFS: Implement FileSystem.access() method.
new aa9cd0a HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy
Configurable.
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hadoop-tools/hadoop-azure/pom.xml | 7 +-
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 70 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 26 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 865 +
.../fs/azurebfs/constants/AbfsHttpConstants.java | 1 +
.../AuthConfigurations.java} | 29 +-
.../fs/azurebfs/constants/ConfigurationKeys.java | 20 +
.../constants/FileSystemConfigurations.java| 5 +-
.../constants/HttpHeaderConfigurations.java| 1 +
.../fs/azurebfs/constants/HttpQueryParams.java | 1 +
.../contracts/services/AbfsPerfLoggable.java | 24 +-
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 23 +-
.../fs/azurebfs/oauth2/MsiTokenProvider.java | 14 +-
.../oauth2/RefreshTokenBasedTokenProvider.java | 12 +-
.../hadoop/fs/azurebfs/services/AbfsClient.java| 31 +-
.../fs/azurebfs/services/AbfsHttpOperation.java| 46 +-
.../fs/azurebfs/services/AbfsInputStream.java | 4 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 28 +-
.../hadoop/fs/azurebfs/services/AbfsPerfInfo.java | 133
.../fs/azurebfs/services/AbfsPerfTracker.java | 319
.../fs/azurebfs/services/AbfsRestOperation.java| 11 +
.../azurebfs/services/AbfsRestOperationType.java | 3 +-
.../azurebfs/services/ExponentialRetryPolicy.java | 11 +-
.../hadoop-azure/src/site/markdown/abfs.md | 67 ++
.../azurebfs/ITestAbfsRestOperationException.java | 41 +
.../ITestAzureBlobFileSystemCheckAccess.java | 314
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 +-
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../azurebfs/constants/TestConfigurationKeys.java | 8 +
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 67 ++
.../fs/azurebfs/services/TestAbfsClient.java | 2 +-
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 408 ++
.../hadoop-azure/src/test/resources/azure-test.xml | 5 +
34 files changed, insertions(+), 409 deletions(-)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{contracts/exceptions/InvalidAbfsRestOperationException.java
=> constants/AuthConfigurations.java} (55%)
copy
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/utils/package-info.java
=>
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AbfsPerfLoggable.java
(75%)
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPerfInfo.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPerfTracker.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/06: HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8b2c7e0c4d87eb792c573b16e3721e5e4020752f
Author: Sneha Vijayarajan
AuthorDate: Tue Oct 1 17:38:11 2019 -0700
HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
---
.../java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 3 ++-
.../org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 --
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 1663ed9..82a38a2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
@@ -107,7 +108,7 @@ public class AzureBlobFileSystem extends FileSystem {
this.setWorkingDirectory(this.getHomeDirectory());
if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) {
- if (!this.fileSystemExists()) {
+ if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH)) ==
null) {
try {
this.createFileSystem();
} catch (AzureBlobFileSystemException ex) {
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index b9a2315..74c8803 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.UUID;
@@ -24,6 +25,7 @@ import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
@@ -67,10 +69,10 @@ public class ITestGetNameSpaceEnabled extends
AbstractAbfsIntegrationTest {
+ testUri.substring(testUri.indexOf("@"));
AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl);
-intercept(AbfsRestOperationException.class,
+intercept(FileNotFoundException.class,
"\"The specified filesystem does not exist.\", 404",
()-> {
- fs.getIsNamespaceEnabled();
+ fs.getFileStatus(new Path("/")); // Run a dummy FS call
});
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy Configurable.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 82ad9b5 HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy
Configurable.
82ad9b5 is described below
commit 82ad9b549f9ae0867699042e0f5856577919a749
Author: Sneha Vijayarajan
AuthorDate: Wed Nov 27 15:07:27 2019 -0800
HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy Configurable.
Contributed by Sneha Vijayarajan.
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +-
.../fs/azurebfs/services/AbfsRestOperation.java| 3 +
.../azurebfs/services/ExponentialRetryPolicy.java | 11 ++--
.../azurebfs/ITestAbfsRestOperationException.java | 41 +
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 67 ++
6 files changed, 120 insertions(+), 10 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 031a20a..36ab57c 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -1129,7 +1129,9 @@ public class AzureBlobFileSystemStore implements
Closeable {
abfsConfiguration.getRawConfiguration());
}
-this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, new
ExponentialRetryPolicy(), tokenProvider, abfsPerfTracker);
+this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
+new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()),
+tokenProvider, abfsPerfTracker);
}
private String getOctalNotation(FsPermission fsPermission) {
@@ -1353,4 +1355,4 @@ public class AzureBlobFileSystemStore implements
Closeable {
AbfsClient getClient() {
return this.client;
}
-}
+}
\ No newline at end of file
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index a171068..f44094b 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -130,8 +130,11 @@ public class AbfsRestOperation {
}
int retryCount = 0;
+LOG.debug("First execution of REST operation - {}", operationType);
while (!executeHttpOperation(retryCount++)) {
try {
+LOG.debug("Retrying REST operation {}. RetryCount = {}",
+operationType, retryCount);
Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
index 5eb7a66..b272cf2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
@@ -26,11 +26,6 @@ import java.net.HttpURLConnection;
* */
public class ExponentialRetryPolicy {
/**
- * Represents the default number of retry attempts.
- */
- private static final int DEFAULT_CLIENT_RETRY_COUNT = 30;
-
- /**
* Represents the default amount of time used when calculating a random
delta in the exponential
* delay between retries.
*/
@@ -86,8 +81,10 @@ public class ExponentialRetryPolicy {
/**
* Initializes a new instance of the {@link ExponentialRetryPolicy} class.
*/
- public ExponentialRetryPolicy() {
-this(DEFAULT_CLIENT_RETRY_COUNT, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
DEFAULT_CLIENT_BACKOFF);
+ public ExponentialRetryPolicy(final int maxIoRetries) {
+
+this(maxIoRetries, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
+DEFAULT_CLIENT_BACKOFF);
}
/**
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index ff88b02..6f69826 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apach
[hadoop] branch trunk updated: HADOOP-16612. Track Azure Blob File System client-perceived latency
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b033c68 HADOOP-16612. Track Azure Blob File System client-perceived
latency
b033c68 is described below
commit b033c681e4fc3ee1a38caa807e130aee481d99d5
Author: Jeetesh Mangwani
AuthorDate: Tue Nov 19 08:51:49 2019 -0800
HADOOP-16612. Track Azure Blob File System client-perceived latency
Contributed by Jeetesh Mangwani.
This add the ability to track the end-to-end performance of ADLS Gen 2 REST
APIs by measuring latency in the Hadoop ABFS driver.
The latency information is sent back to the ADLS Gen 2 REST API endpoints
in the subsequent requests.
---
hadoop-tools/hadoop-azure/pom.xml | 5 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 13 +
.../fs/azurebfs/AzureBlobFileSystemStore.java | 844 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 2 +
.../constants/FileSystemConfigurations.java| 1 +
.../constants/HttpHeaderConfigurations.java| 1 +
.../contracts/services/AbfsPerfLoggable.java | 34 +
.../hadoop/fs/azurebfs/services/AbfsClient.java| 9 +-
.../fs/azurebfs/services/AbfsHttpOperation.java| 46 +-
.../fs/azurebfs/services/AbfsInputStream.java | 4 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 21 +-
.../hadoop/fs/azurebfs/services/AbfsPerfInfo.java | 133
.../fs/azurebfs/services/AbfsPerfTracker.java | 319
.../fs/azurebfs/services/AbfsRestOperation.java| 8 +
.../hadoop-azure/src/site/markdown/abfs.md | 46 ++
.../fs/azurebfs/services/TestAbfsClient.java | 2 +-
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 408 ++
.../hadoop-azure/src/test/resources/azure-test.xml | 5 +
18 files changed, 1561 insertions(+), 340 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml
b/hadoop-tools/hadoop-azure/pom.xml
index 1a4250f..f2af5a9 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -278,6 +278,11 @@
bcpkix-jdk15on
test
+
+ org.assertj
+ assertj-core
+ test
+
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index ef2d83f..9b8c156 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -178,6 +178,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_USE_UPN)
private boolean useUpn;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ABFS_LATENCY_TRACK,
+ DefaultValue = DEFAULT_ABFS_LATENCY_TRACK)
+ private boolean trackLatency;
+
private Map storageAccountKeys;
public AbfsConfiguration(final Configuration rawConfig, String accountName)
@@ -471,6 +475,15 @@ public class AbfsConfiguration{
return this.useUpn;
}
+ /**
+ * Whether {@code AbfsClient} should track and send latency info back to
storage servers.
+ *
+ * @return a boolean indicating whether latency should be tracked.
+ */
+ public boolean shouldTrackLatency() {
+return this.trackLatency;
+ }
+
public AccessTokenProvider getTokenProvider() throws
TokenAccessProviderException {
AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
AuthType.SharedKey);
if (authType == AuthType.OAuth) {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 7f1bf10..51953d4 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -35,6 +35,7 @@ import java.nio.charset.CharsetEncoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -82,6 +83,8 @@ import
org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
+import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker;
+import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import
[hadoop] branch branch-2 updated: HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 7f71473 HADOOP-16640. WASB: Override getCanonicalServiceName() to
return URI
7f71473 is described below
commit 7f71473888f1c06b22083038e6c398252fdf42d0
Author: Da Zhou
AuthorDate: Wed Oct 16 13:14:15 2019 -0700
HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
(cherry picked from commit 9a8edb0aeddd7787b2654f6e2a8465c325e048a2)
---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 22
.../fs/azure/ITestWasbUriAndConfiguration.java | 61 --
2 files changed, 67 insertions(+), 16 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 11a110b..41b1306 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -640,6 +640,20 @@ public class NativeAzureFileSystem extends FileSystem {
return "wasb";
}
+ /**
+ * If fs.azure.override.canonical.service.name is set as true, return URI of
+ * the WASB filesystem, otherwise use the default implementation.
+ *
+ * @return a service string that uniquely identifies this file system
+ */
+ @Override
+ public String getCanonicalServiceName() {
+if (returnUriAsCanonicalServiceName) {
+ return getUri().toString();
+}
+return super.getCanonicalServiceName();
+ }
+
/**
*
@@ -723,6 +737,11 @@ public class NativeAzureFileSystem extends FileSystem {
*/
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME =
"fs.azure.enable.append.support";
+ /*
+ * Property to override canonical service name with filesystem's URI.
+ */
+ public static final String
RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME =
"fs.azure.override.canonical.service.name";
+
/**
* The configuration property to set number of threads to be used for rename
operation.
*/
@@ -1198,6 +1217,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
+ private boolean returnUriAsCanonicalServiceName = false;
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new
DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
@@ -1395,6 +1415,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
this.wasbDelegationTokenManager = new
RemoteWasbDelegationTokenManager(conf);
}
+
+this.returnUriAsCanonicalServiceName =
conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
}
@Override
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
index 7783684..62a751d 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
@@ -18,30 +18,16 @@
package org.apache.hadoop.fs.azure;
-import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
-import static org.junit.Assume.assumeNotNull;
-
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.Date;
import java.util.EnumSet;
-import java.io.File;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
@@ -49,10 +35,24 @@ import org.junit.Before;
import org.
[hadoop] branch branch-2 updated: HADOOP-16652. Backport of HADOOP-16587: Make ABFS AAD endpoints configurable
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 6cb9ee3 HADOOP-16652. Backport of HADOOP-16587: Make ABFS AAD
endpoints configurable
6cb9ee3 is described below
commit 6cb9ee32e11ab968492739a4e2cb00c74d43dd53
Author: Bilahari T H
AuthorDate: Wed Oct 16 15:39:39 2019 -0700
HADOOP-16652. Backport of HADOOP-16587: Make ABFS AAD endpoints configurable
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 35 -
.../fs/azurebfs/constants/AuthConfigurations.java | 45 ++
.../fs/azurebfs/constants/ConfigurationKeys.java | 6 +++
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 23 ++-
.../fs/azurebfs/oauth2/MsiTokenProvider.java | 14 +--
.../oauth2/RefreshTokenBasedTokenProvider.java | 12 --
6 files changed, 118 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 56ff622..ffddc45 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -25,9 +25,12 @@ import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
+import org.apache.hadoop.fs.azurebfs.constants.AuthConfigurations;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
import
org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@@ -69,6 +72,7 @@ import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class AbfsConfiguration{
+
private final Configuration rawConfig;
private final String accountName;
private final boolean isSecure;
@@ -486,13 +490,25 @@ public class AbfsConfiguration{
String password =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD);
tokenProvider = new UserPasswordTokenProvider(authEndpoint,
username, password);
} else if (tokenProviderClass == MsiTokenProvider.class) {
+ String authEndpoint = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
+ AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT);
String tenantGuid =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT);
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
- tokenProvider = new MsiTokenProvider(tenantGuid, clientId);
+ String authority = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY,
+ AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY);
+ authority = appendSlashIfNeeded(authority);
+ tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid,
+ clientId, authority);
} else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class)
{
+ String authEndpoint = getTrimmedPasswordString(
+ FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT,
+
AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT);
String refreshToken =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN);
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
- tokenProvider = new RefreshTokenBasedTokenProvider(clientId,
refreshToken);
+ tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint,
+ clientId, refreshToken);
} else {
throw new IllegalArgumentException("Failed to initialize " +
tokenProviderClass);
}
@@ -649,4 +665,19 @@ public class AbfsConfiguration{
this.disableOutputStreamFlush = disableOutputStreamFlush;
}
+ private String getTrimmedPasswordString(String key, String defaultValue)
throws IOException {
+String value = getPasswordString(key);
+if (StringUtils.isBlank(value)) {
+ value = defaultValue;
+}
+return value.trim();
+ }
+
+ private String appendSlashIfNeeded(String authority) {
+if (!authorit
[hadoop] branch branch-3.2 updated: HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new fe96407 HADOOP-16640. WASB: Override getCanonicalServiceName() to
return URI
fe96407 is described below
commit fe9640745163aae9b910d18bc33d1fa6c06b968a
Author: Da Zhou
AuthorDate: Wed Oct 16 13:14:15 2019 -0700
HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
(cherry picked from commit 9a8edb0aeddd7787b2654f6e2a8465c325e048a2)
---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 22
.../fs/azure/ITestWasbUriAndConfiguration.java | 30 +-
2 files changed, 51 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index f8962d9..e8e867e 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -641,6 +641,20 @@ public class NativeAzureFileSystem extends FileSystem {
return "wasb";
}
+ /**
+ * If fs.azure.override.canonical.service.name is set as true, return URI of
+ * the WASB filesystem, otherwise use the default implementation.
+ *
+ * @return a service string that uniquely identifies this file system
+ */
+ @Override
+ public String getCanonicalServiceName() {
+if (returnUriAsCanonicalServiceName) {
+ return getUri().toString();
+}
+return super.getCanonicalServiceName();
+ }
+
/**
*
@@ -724,6 +738,11 @@ public class NativeAzureFileSystem extends FileSystem {
*/
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME =
"fs.azure.enable.append.support";
+ /*
+ * Property to override canonical service name with filesystem's URI.
+ */
+ public static final String
RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME =
"fs.azure.override.canonical.service.name";
+
/**
* The configuration property to set number of threads to be used for rename
operation.
*/
@@ -1190,6 +1209,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
+ private boolean returnUriAsCanonicalServiceName = false;
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new
DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
@@ -1387,6 +1407,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
this.wasbDelegationTokenManager = new
RemoteWasbDelegationTokenManager(conf);
}
+
+this.returnUriAsCanonicalServiceName =
conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
}
@Override
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
index 19d329a..982e92b 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azure;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static
org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeNotNull;
@@ -44,7 +45,6 @@ import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.apache.hadoop.test.GenericTestUtils;
@@ -640,4 +640,32 @@ public class ITestWasbUriAndConfiguration extends
AbstractWasbTestWithTimeout {
FileSystem.closeAll();
}
}
+
+ @Test
+ public void testCanonicalServiceName() throws Exception {
+AzureBlobStorageTestAccount testAccount =
AzureBlobStorageTestAccount.createMock();
+Configuration conf = testAccount.getFileSystem().getConf();
+String authority = testAccount.getFil
[hadoop] branch trunk updated: HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9a8edb0 HADOOP-16640. WASB: Override getCanonicalServiceName() to
return URI
9a8edb0 is described below
commit 9a8edb0aeddd7787b2654f6e2a8465c325e048a2
Author: Da Zhou
AuthorDate: Wed Oct 16 13:14:15 2019 -0700
HADOOP-16640. WASB: Override getCanonicalServiceName() to return URI
---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 22
.../fs/azure/ITestWasbUriAndConfiguration.java | 30 +-
2 files changed, 51 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index a990b60..93c54d3 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -643,6 +643,20 @@ public class NativeAzureFileSystem extends FileSystem {
return "wasb";
}
+ /**
+ * If fs.azure.override.canonical.service.name is set as true, return URI of
+ * the WASB filesystem, otherwise use the default implementation.
+ *
+ * @return a service string that uniquely identifies this file system
+ */
+ @Override
+ public String getCanonicalServiceName() {
+if (returnUriAsCanonicalServiceName) {
+ return getUri().toString();
+}
+return super.getCanonicalServiceName();
+ }
+
/**
*
@@ -726,6 +740,11 @@ public class NativeAzureFileSystem extends FileSystem {
*/
public static final String APPEND_SUPPORT_ENABLE_PROPERTY_NAME =
"fs.azure.enable.append.support";
+ /*
+ * Property to override canonical service name with filesystem's URI.
+ */
+ public static final String
RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME =
"fs.azure.override.canonical.service.name";
+
/**
* The configuration property to set number of threads to be used for rename
operation.
*/
@@ -1192,6 +1211,7 @@ public class NativeAzureFileSystem extends FileSystem {
// A counter to create unique (within-process) names for my metrics sources.
private static AtomicInteger metricsSourceNameCounter = new AtomicInteger();
private boolean appendSupportEnabled = false;
+ private boolean returnUriAsCanonicalServiceName = false;
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new
DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
@@ -1389,6 +1409,8 @@ public class NativeAzureFileSystem extends FileSystem {
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
this.wasbDelegationTokenManager = new
RemoteWasbDelegationTokenManager(conf);
}
+
+this.returnUriAsCanonicalServiceName =
conf.getBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, false);
}
@Override
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
index 19d329a..982e92b 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azure;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static
org.apache.hadoop.fs.azure.NativeAzureFileSystem.RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeNotNull;
@@ -44,7 +45,6 @@ import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.apache.hadoop.test.GenericTestUtils;
@@ -640,4 +640,32 @@ public class ITestWasbUriAndConfiguration extends
AbstractWasbTestWithTimeout {
FileSystem.closeAll();
}
}
+
+ @Test
+ public void testCanonicalServiceName() throws Exception {
+AzureBlobStorageTestAccount testAccount =
AzureBlobStorageTestAccount.createMock();
+Configuration conf = testAccount.getFileSystem().getConf();
+String authority = testAccount.getFileSystem().getUri().getAuthority();
+URI defaultUri = new URI("wasbs", authority,
[hadoop] branch branch-2 updated: HADOOP-16630 : Backport of Hadoop-16548 : Disable Flush() over config
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 27d7d09 HADOOP-16630 : Backport of Hadoop-16548 : Disable Flush()
over config
27d7d09 is described below
commit 27d7d09fa48afe4cccd766a5567464b772a46300
Author: Sneha Vijayarajan
AuthorDate: Wed Oct 9 16:50:06 2019 -0700
HADOOP-16630 : Backport of Hadoop-16548 : Disable Flush() over config
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 +++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +++--
.../fs/azurebfs/constants/ConfigurationKeys.java | 8 +++
.../constants/FileSystemConfigurations.java| 1 +
.../fs/azurebfs/services/AbfsOutputStream.java | 7 --
.../hadoop-azure/src/site/markdown/abfs.md | 18 +++
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +++---
7 files changed, 64 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 67055c5..56ff622 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -146,6 +146,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_ENABLE_FLUSH)
private boolean enableFlush;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH,
+ DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH)
+ private boolean disableOutputStreamFlush;
+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ENABLE_AUTOTHROTTLING,
DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING)
private boolean enableAutoThrottling;
@@ -427,6 +431,10 @@ public class AbfsConfiguration{
return this.enableFlush;
}
+ public boolean isOutputStreamFlushDisabled() {
+return this.disableOutputStreamFlush;
+ }
+
public boolean isAutoThrottlingEnabled() {
return this.enableAutoThrottling;
}
@@ -635,4 +643,10 @@ public class AbfsConfiguration{
void setEnableFlush(boolean enableFlush) {
this.enableFlush = enableFlush;
}
+
+ @VisibleForTesting
+ void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) {
+this.disableOutputStreamFlush = disableOutputStreamFlush;
+ }
+
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 2694565..6710275 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -354,7 +354,8 @@ public class AzureBlobFileSystemStore {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
0,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void createDirectory(final Path path, final FsPermission permission,
final FsPermission umask)
@@ -426,7 +427,8 @@ public class AzureBlobFileSystemStore {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
offset,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void rename(final Path source, final Path destination) throws
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 8cd86bf..cd86f18 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -51,7 +51,15 @@ public final class ConfigurationKeys {
public static final String FS_AZURE_ALWAYS_USE_HTTPS =
"fs.azure.always.use.https";
public static final String FS_AZURE_ATOMIC_RENAME_KEY =
"fs.azure.atomic.rename.key";
public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH =
"fs.azure.readaheadqueue.depth";
+ /** Provides a config control to enable or disable ABFS Flush operations -
+ * HFlush and HSync. Default is true. **/
public static final String FS_AZURE_ENABLE_FLUSH = "fs.azu
[hadoop] branch branch-2 updated: HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 2ccb168 HADOOP-16578 : Avoid FileSystem API calls when FileSystem
already exists
2ccb168 is described below
commit 2ccb1683ba00a55991970af75240fd064e819fc5
Author: Sneha Vijayarajan
AuthorDate: Tue Oct 1 17:38:11 2019 -0700
HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
---
.../java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 3 ++-
.../org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 --
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 1663ed9..82a38a2 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
@@ -107,7 +108,7 @@ public class AzureBlobFileSystem extends FileSystem {
this.setWorkingDirectory(this.getHomeDirectory());
if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) {
- if (!this.fileSystemExists()) {
+ if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH)) ==
null) {
try {
this.createFileSystem();
} catch (AzureBlobFileSystemException ex) {
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index 1215756..a7d1af8 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.UUID;
@@ -25,6 +26,7 @@ import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
@@ -68,12 +70,12 @@ public class ITestGetNameSpaceEnabled extends
AbstractAbfsIntegrationTest {
+ testUri.substring(testUri.indexOf("@"));
final AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl);
-intercept(AbfsRestOperationException.class,
+intercept(FileNotFoundException.class,
"\"The specified filesystem does not exist.\", 404",
new LambdaTestUtils.VoidCallable() {
@Override
public void call() throws Exception {
-fs.getIsNamespaceEnabled();
+fs.getFileStatus(new Path("/")); // Run a dummy FS call
}
});
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 770adc5 HADOOP-16578 : Avoid FileSystem API calls when FileSystem
already exists
770adc5 is described below
commit 770adc5d4abd71c58812066cf691fc565efea64c
Author: Sneha Vijayarajan
AuthorDate: Tue Oct 1 17:38:11 2019 -0700
HADOOP-16578 : Avoid FileSystem API calls when FileSystem already exists
---
.../java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 3 ++-
.../org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 --
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 2a4e76b..c6640c4 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
@@ -111,7 +112,7 @@ public class AzureBlobFileSystem extends FileSystem {
this.setWorkingDirectory(this.getHomeDirectory());
if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) {
- if (!this.fileSystemExists()) {
+ if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH)) ==
null) {
try {
this.createFileSystem();
} catch (AzureBlobFileSystemException ex) {
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
index b9a2315..74c8803 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.UUID;
@@ -24,6 +25,7 @@ import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
@@ -67,10 +69,10 @@ public class ITestGetNameSpaceEnabled extends
AbstractAbfsIntegrationTest {
+ testUri.substring(testUri.indexOf("@"));
AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl);
-intercept(AbfsRestOperationException.class,
+intercept(FileNotFoundException.class,
"\"The specified filesystem does not exist.\", 404",
()-> {
- fs.getIsNamespaceEnabled();
+ fs.getFileStatus(new Path("/")); // Run a dummy FS call
});
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16548 : Disable Flush() over config
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c0edc84 HADOOP-16548 : Disable Flush() over config
c0edc84 is described below
commit c0edc848a8e71b5e2a1586a589bbf2ac8685040d
Author: Sneha Vijayarajan
AuthorDate: Sat Sep 28 20:39:42 2019 -0700
HADOOP-16548 : Disable Flush() over config
---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 +++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +++--
.../fs/azurebfs/constants/ConfigurationKeys.java | 8 +++
.../constants/FileSystemConfigurations.java| 1 +
.../fs/azurebfs/services/AbfsOutputStream.java | 7 --
.../hadoop-azure/src/site/markdown/abfs.md | 17 ++
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +++---
7 files changed, 63 insertions(+), 17 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 19b1a90..723a25c 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -146,6 +146,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_ENABLE_FLUSH)
private boolean enableFlush;
+ @BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH,
+ DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH)
+ private boolean disableOutputStreamFlush;
+
@BooleanConfigurationValidatorAnnotation(ConfigurationKey =
FS_AZURE_ENABLE_AUTOTHROTTLING,
DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING)
private boolean enableAutoThrottling;
@@ -427,6 +431,10 @@ public class AbfsConfiguration{
return this.enableFlush;
}
+ public boolean isOutputStreamFlushDisabled() {
+return this.disableOutputStreamFlush;
+ }
+
public boolean isAutoThrottlingEnabled() {
return this.enableAutoThrottling;
}
@@ -635,4 +643,10 @@ public class AbfsConfiguration{
void setEnableFlush(boolean enableFlush) {
this.enableFlush = enableFlush;
}
+
+ @VisibleForTesting
+ void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) {
+this.disableOutputStreamFlush = disableOutputStreamFlush;
+ }
+
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 6b2d196..7f1bf10 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -362,7 +362,8 @@ public class AzureBlobFileSystemStore implements Closeable {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
0,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void createDirectory(final Path path, final FsPermission permission,
final FsPermission umask)
@@ -434,7 +435,8 @@ public class AzureBlobFileSystemStore implements Closeable {
AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
offset,
abfsConfiguration.getWriteBufferSize(),
-abfsConfiguration.isFlushEnabled());
+abfsConfiguration.isFlushEnabled(),
+abfsConfiguration.isOutputStreamFlushDisabled());
}
public void rename(final Path source, final Path destination) throws
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 8cd86bf..cd86f18 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -51,7 +51,15 @@ public final class ConfigurationKeys {
public static final String FS_AZURE_ALWAYS_USE_HTTPS =
"fs.azure.always.use.https";
public static final String FS_AZURE_ATOMIC_RENAME_KEY =
"fs.azure.atomic.rename.key";
public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH =
"fs.azure.readaheadqueue.depth";
+ /** Provides a config control to enable or disable ABFS Flush operations -
+ * HFlush and HSync. Default is true. **/
public static final String FS_AZURE_ENABLE_FLUSH = "fs.azure.enable.flush&qu
[hadoop] branch branch-3.2 updated: HADOOP-16315. ABFS: transform full UPN for named user in AclStatus
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 330e450 HADOOP-16315. ABFS: transform full UPN for named user in
AclStatus
330e450 is described below
commit 330e450397b0031272f89ed964cb593ba2a490d4
Author: Da Zhou
AuthorDate: Fri Aug 9 12:37:27 2019 +0100
HADOOP-16315. ABFS: transform full UPN for named user in AclStatus
Contributed by Da Zhou
Change-Id: Ibc78322415fcbeff89c06c8586c53f5695550290
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 17 ++---
.../fs/azurebfs/oauth2/IdentityTransformer.java| 75 +++---
.../fs/azurebfs/ITestAbfsIdentityTransformer.java | 58 -
3 files changed, 131 insertions(+), 19 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index f1ece36..bfab487 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -624,8 +624,8 @@ public class AzureBlobFileSystemStore {
path.toString(),
AclEntry.aclSpecToString(aclSpec));
-final List transformedAclEntries =
identityTransformer.transformAclEntriesForSetRequest(aclSpec);
-final Map modifyAclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(transformedAclEntries));
+identityTransformer.transformAclEntriesForSetRequest(aclSpec);
+final Map modifyAclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(aclSpec));
boolean useUpn = AbfsAclHelper.isUpnFormatAclEntries(modifyAclEntries);
final AbfsRestOperation op =
client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path,
true), useUpn);
@@ -651,8 +651,8 @@ public class AzureBlobFileSystemStore {
path.toString(),
AclEntry.aclSpecToString(aclSpec));
-final List transformedAclEntries =
identityTransformer.transformAclEntriesForSetRequest(aclSpec);
-final Map removeAclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(transformedAclEntries));
+identityTransformer.transformAclEntriesForSetRequest(aclSpec);
+final Map removeAclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(aclSpec));
boolean isUpnFormat =
AbfsAclHelper.isUpnFormatAclEntries(removeAclEntries);
final AbfsRestOperation op =
client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path,
true), isUpnFormat);
@@ -730,8 +730,8 @@ public class AzureBlobFileSystemStore {
path.toString(),
AclEntry.aclSpecToString(aclSpec));
-final List transformedAclEntries =
identityTransformer.transformAclEntriesForSetRequest(aclSpec);
-final Map aclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(transformedAclEntries));
+identityTransformer.transformAclEntriesForSetRequest(aclSpec);
+final Map aclEntries =
AbfsAclHelper.deserializeAclSpec(AclEntry.aclSpecToString(aclSpec));
final boolean isUpnFormat =
AbfsAclHelper.isUpnFormatAclEntries(aclEntries);
final AbfsRestOperation op =
client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path,
true), isUpnFormat);
@@ -770,7 +770,8 @@ public class AzureBlobFileSystemStore {
final String permissions =
result.getResponseHeader(HttpHeaderConfigurations.X_MS_PERMISSIONS);
final String aclSpecString =
op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_ACL);
-final List processedAclEntries =
AclEntry.parseAclSpec(AbfsAclHelper.processAclString(aclSpecString), true);
+final List aclEntries =
AclEntry.parseAclSpec(AbfsAclHelper.processAclString(aclSpecString), true);
+identityTransformer.transformAclEntriesForGetRequest(aclEntries, userName,
primaryUserGroup);
final FsPermission fsPermission = permissions == null ? new
AbfsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)
: AbfsPermission.valueOf(permissions);
@@ -780,7 +781,7 @@ public class AzureBlobFileSystemStore {
aclStatusBuilder.setPermission(fsPermission);
aclStatusBuilder.stickyBit(fsPermission.getStickyBit());
-aclStatusBuilder.addEntries(processedAclEntries);
+aclStatusBuilder.addEntries(aclEntries);
return aclStatusBuilder.build();
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java
index 343b233..6844afb 100644
---
a/hadoop-tools/hadoop-azure/src/main/java
[hadoop] branch branch-3.2 updated: HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5f2d07a HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
5f2d07a is described below
commit 5f2d07af1bdfe4a482d98af1504008e0ae9408be
Author: Arun Singh
AuthorDate: Fri Jul 19 20:21:28 2019 -0700
HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
Contributed by: Arun Singh
---
.../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +-
.../hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java | 6 ++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 9744307..a2a0064 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -44,7 +44,7 @@ public final class FileSystemConfigurations {
public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB
public static final int MIN_BUFFER_SIZE = 16 * ONE_KB; // 16 KB
public static final int MAX_BUFFER_SIZE = 100 * ONE_MB; // 100 MB
- public static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L;
+ public static final long MAX_AZURE_BLOCK_SIZE = 256 * 1024 * 1024L; //
changing default abfs blocksize to 256MB
public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost";
public static final int MAX_CONCURRENT_READ_THREADS = 12;
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index eeed6ce..9848228 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -144,6 +144,12 @@ public class TestAbfsConfigurationFieldsValidation {
}
@Test
+ public void testConfigBlockSizeInitialized() throws Exception {
+// test the block size annotated field has been initialized in the
constructor
+assertEquals(MAX_AZURE_BLOCK_SIZE, abfsConfiguration.getAzureBlockSize());
+ }
+
+ @Test
public void testGetAccountKey() throws Exception {
String accountKey = abfsConfiguration.getStorageAccountKey();
assertEquals(this.encodedAccountKey, accountKey);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0b45293 HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
0b45293 is described below
commit 0b45293abb3a58066bca7f668c3b4c0e2d006a6c
Author: Arun Singh
AuthorDate: Fri Jul 19 20:21:28 2019 -0700
HADOOP-16404. ABFS default blocksize change(256MB from 512MB)
Contributed by: Arun Singh
---
.../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +-
.../hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java | 6 ++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 9744307..a2a0064 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -44,7 +44,7 @@ public final class FileSystemConfigurations {
public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB
public static final int MIN_BUFFER_SIZE = 16 * ONE_KB; // 16 KB
public static final int MAX_BUFFER_SIZE = 100 * ONE_MB; // 100 MB
- public static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L;
+ public static final long MAX_AZURE_BLOCK_SIZE = 256 * 1024 * 1024L; //
changing default abfs blocksize to 256MB
public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost";
public static final int MAX_CONCURRENT_READ_THREADS = 12;
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index eeed6ce..9848228 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -144,6 +144,12 @@ public class TestAbfsConfigurationFieldsValidation {
}
@Test
+ public void testConfigBlockSizeInitialized() throws Exception {
+// test the block size annotated field has been initialized in the
constructor
+assertEquals(MAX_AZURE_BLOCK_SIZE, abfsConfiguration.getAzureBlockSize());
+ }
+
+ @Test
public void testGetAccountKey() throws Exception {
String accountKey = abfsConfiguration.getStorageAccountKey();
assertEquals(this.encodedAccountKey, accountKey);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
