[hadoop] branch trunk updated (b56af00 -> 9199787)
This is an automated email from the ASF dual-hosted git repository. ferhui pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b56af00 HADOOP-18075. ABFS: Fix failure caused by listFiles() in ITestAbfsRestOperationException (#4040) add 9199787 HDFS-16458. [SPS]: Fix bug for unit test of reconfiguring SPS mode (#4041) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a3d8749 YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
a3d8749 is described below
commit a3d8749b8ca5646bd7f6b272673452d3427ad9d7
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:08:38 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1cbe5df..abb7c3d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 4bd67d7 YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
4bd67d7 is described below
commit 4bd67d751663cc76d68d08af69bd22ac64900a92
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:06:39 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1d5a6ac..2e29f28 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 935619a YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
935619a is described below
commit 935619a28c0a81d99de7e46f2b8ccf5d552b0453
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:05:04 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4480bc3..425d63f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
public static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 03/03: HADOOP-18075. ABFS: Fix failure caused by listFiles() in ITestAbfsRestOperationException (#4040)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 36a50ba3e06207d852d1f1728d82bc23e6baf1f7
Author: Steve Loughran
AuthorDate: Tue Mar 1 11:48:10 2022 +
HADOOP-18075. ABFS: Fix failure caused by listFiles() in
ITestAbfsRestOperationException (#4040)
Contributed by Sumangala Patki
Change-Id: I245c08dab050d59b90ac6fdcb4c03153db77be0b
---
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 8 ++---
.../services/AbfsListStatusRemoteIterator.java | 36 ++
.../ITestAbfsListStatusRemoteIterator.java | 34 +---
.../azurebfs/ITestAbfsRestOperationException.java | 5 +--
4 files changed, 45 insertions(+), 38 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 750306c..86bd904 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -1185,7 +1185,7 @@ public class AzureBlobFileSystem extends FileSystem
TracingContext tracingContext = new TracingContext(clientCorrelationId,
fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat,
listener);
AbfsListStatusRemoteIterator abfsLsItr =
- new AbfsListStatusRemoteIterator(getFileStatus(path,
tracingContext), abfsStore,
+ new AbfsListStatusRemoteIterator(path, abfsStore,
tracingContext);
return RemoteIterators.typeCastingRemoteIterator(abfsLsItr);
} else {
@@ -1360,9 +1360,9 @@ public class AzureBlobFileSystem extends FileSystem
* @throws IOException if the exception error code is not on the allowed
list.
*/
@VisibleForTesting
- static void checkException(final Path path,
- final AzureBlobFileSystemException exception,
- final AzureServiceErrorCode...
allowedErrorCodesList) throws IOException {
+ public static void checkException(final Path path,
+ final AzureBlobFileSystemException exception,
+ final AzureServiceErrorCode... allowedErrorCodesList) throws IOException
{
if (exception instanceof AbfsRestOperationException) {
AbfsRestOperationException ere = (AbfsRestOperationException) exception;
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
index ce6207b..3fecb1f 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
@@ -32,7 +32,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
public class AbfsListStatusRemoteIterator
@@ -45,7 +48,7 @@ public class AbfsListStatusRemoteIterator
private static final int MAX_QUEUE_SIZE = 10;
private static final long POLL_WAIT_TIME_IN_MS = 250;
- private final FileStatus fileStatus;
+ private final Path path;
private final ListingSupport listingSupport;
private final ArrayBlockingQueue listResultQueue;
private final TracingContext tracingContext;
@@ -55,13 +58,15 @@ public class AbfsListStatusRemoteIterator
private String continuation;
private Iterator currIterator;
- public AbfsListStatusRemoteIterator(final FileStatus fileStatus,
- final ListingSupport listingSupport, TracingContext tracingContext) {
-this.fileStatus = fileStatus;
+ public AbfsListStatusRemoteIterator(final Path path,
+ final ListingSupport listingSupport, TracingContext tracingContext)
+ throws IOException {
+this.path = path;
this.listingSupport = listingSupport;
this.tracingContext = tracingContext;
listResultQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
currIterator = Collections.emptyIterator();
+addNextBatchIteratorToQueue();
fetchBatchesAsync();
}
@@ -130,9 +135,6 @@ public class AbfsListStatusRemoteIterator
Thread.currentThread().interrupt();
LOG.error("Thread got interrupted: {}", interruptedException);
}
-} catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("Thread got
[hadoop] 02/03: HADOOP-17862. ABFS: Fix unchecked cast compiler warning for AbfsListStatusRemoteIterator (#3331)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0ed037541316feb9e431bb4e69e53c239aa2decd
Author: sumangala-patki <70206833+sumangala-pa...@users.noreply.github.com>
AuthorDate: Fri Nov 5 18:20:37 2021 +0530
HADOOP-17862. ABFS: Fix unchecked cast compiler warning for
AbfsListStatusRemoteIterator (#3331)
closes #3331
Contributed by Sumangala Patki
Change-Id: I6cca91c8bcc34052c5233035f14a576f23086067
---
.../fs/azurebfs/services/AbfsListResult.java | 57 +++
.../services/AbfsListStatusRemoteIterator.java | 37 ++--
.../ITestAbfsListStatusRemoteIterator.java | 190 +
3 files changed, 152 insertions(+), 132 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java
new file mode 100644
index 000..0b63a34
--- /dev/null
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * Class to store listStatus results for AbfsListStatusRemoteIterator. The
+ * results can either be of type Iterator or an exception thrown during the
+ * operation
+ */
+public class AbfsListResult {
+ private IOException listException = null;
+
+ private Iterator fileStatusIterator
+ = Collections.emptyIterator();
+
+ AbfsListResult(IOException ex) {
+this.listException = ex;
+ }
+
+ AbfsListResult(Iterator fileStatusIterator) {
+this.fileStatusIterator = fileStatusIterator;
+ }
+
+ IOException getListingException() {
+return listException;
+ }
+
+ Iterator getFileStatusIterator() {
+return fileStatusIterator;
+ }
+
+ boolean isFailedListing() {
+return (listException != null);
+ }
+}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
index 835217f..ce6207b 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
@@ -27,7 +27,6 @@ import java.util.NoSuchElementException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-import javax.activation.UnsupportedDataTypeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,7 @@ public class AbfsListStatusRemoteIterator
private final FileStatus fileStatus;
private final ListingSupport listingSupport;
- private final ArrayBlockingQueue iteratorsQueue;
+ private final ArrayBlockingQueue listResultQueue;
private final TracingContext tracingContext;
private volatile boolean isAsyncInProgress = false;
@@ -61,7 +60,7 @@ public class AbfsListStatusRemoteIterator
this.fileStatus = fileStatus;
this.listingSupport = listingSupport;
this.tracingContext = tracingContext;
-iteratorsQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
+listResultQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
currIterator = Collections.emptyIterator();
fetchBatchesAsync();
}
@@ -86,19 +85,17 @@ public class AbfsListStatusRemoteIterator
private Iterator getNextIterator() throws IOException {
fetchBatchesAsync();
try {
- Object obj = null;
- while (obj == null
- && (!isIterationComplete || !iteratorsQueue.isEmpty())) {
-obj = iteratorsQueue.poll(POLL_WAIT_TIME_IN_MS, TimeUnit.MILLISECONDS);
+ AbfsListResult listResult = null;
+ while (listResult == null
+
[hadoop] 01/03: HADOOP-17765. ABFS: Use Unique File Paths in Tests. (#3153)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5e109705effe6265af48749644d4760e87c15c3c
Author: sumangala-patki <70206833+sumangala-pa...@users.noreply.github.com>
AuthorDate: Tue Jul 27 23:19:22 2021 +0530
HADOOP-17765. ABFS: Use Unique File Paths in Tests. (#3153)
Contributed by Sumangala Patki
Change-Id: Ic8f34bf578069504f7a811a7729982b9c9f49729
---
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 21 ++--
.../apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 11 ++--
.../ITestAbfsListStatusRemoteIterator.java | 6 +--
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 12 +++--
.../hadoop/fs/azurebfs/ITestAbfsStatistics.java| 18 +++
.../fs/azurebfs/ITestAbfsStreamStatistics.java | 4 +-
.../azurebfs/ITestAzureBlobFileSystemAppend.java | 23 +
.../ITestAzureBlobFileSystemAuthorization.java | 6 +--
.../ITestAzureBlobFileSystemBackCompat.java| 9 ++--
.../ITestAzureBlobFileSystemCheckAccess.java | 3 +-
.../fs/azurebfs/ITestAzureBlobFileSystemCopy.java | 2 +-
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 26 ++
.../ITestAzureBlobFileSystemDelegationSAS.java | 34 ++--
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 27 +-
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 59 +++--
.../ITestAzureBlobFileSystemFileStatus.java| 28 +-
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 15 +++---
.../ITestAzureBlobFileSystemListStatus.java| 31 +--
.../fs/azurebfs/ITestAzureBlobFileSystemMkDir.java | 8 +--
.../fs/azurebfs/ITestAzureBlobFileSystemOauth.java | 58 +++--
.../ITestAzureBlobFileSystemPermission.java| 3 +-
.../ITestAzureBlobFileSystemRandomRead.java| 19 +++
.../azurebfs/ITestAzureBlobFileSystemRename.java | 43 +---
.../ITestAzureBlobFileSystemRenameUnicode.java | 2 +-
.../fs/azurebfs/ITestAzureBlobFilesystemAcl.java | 5 +-
.../fs/azurebfs/ITestCustomerProvidedKey.java | 60 +-
.../fs/azurebfs/ITestFileSystemProperties.java | 31 +++
.../fs/azurebfs/ITestWasbAbfsCompatibility.java| 20 +---
.../azurebfs/services/ITestAbfsOutputStream.java | 26 +-
.../fs/azurebfs/services/TestAbfsInputStream.java | 8 +--
30 files changed, 349 insertions(+), 269 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index fd2f269..a57e9bc 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -26,11 +26,11 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
@@ -84,6 +84,7 @@ public abstract class AbstractAbfsIntegrationTest extends
private AuthType authType;
private boolean useConfiguredFileSystem = false;
private boolean usingFilesystemForSASTests = false;
+ private static final int SHORTENED_GUID_LEN = 12;
protected AbstractAbfsIntegrationTest() throws Exception {
fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
@@ -270,7 +271,8 @@ public abstract class AbstractAbfsIntegrationTest extends
// so first create temporary instance of the filesystem using SharedKey
// then re-use the filesystem it creates with SAS auth instead of
SharedKey.
AzureBlobFileSystem tempFs = (AzureBlobFileSystem)
FileSystem.newInstance(rawConfig);
-Assert.assertTrue(tempFs.exists(new Path("/")));
+ContractTestUtils.assertPathExists(tempFs, "This path should exist",
+new Path("/"));
abfsConfig.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
AuthType.SAS.name());
usingFilesystemForSASTests = true;
}
@@ -440,7 +442,20 @@ public abstract class AbstractAbfsIntegrationTest extends
*/
protected Path path(String filepath) throws IOException {
return getFileSystem().makeQualified(
-new Path(getTestPath(), filepath));
+new Path(getTestPath(), getUniquePath(filepath)));
+ }
+
+ /**
+ * Generate a unique path using the given filepath.
+ * @param filepath path string
+ * @return unique path created from filepath and a GUID
+ */
+ protected Path getUniquePath(String filepath) {
+if (filepath.equals("/"))
[hadoop] branch branch-3.3 updated (74f5f90 -> 36a50ba)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 74f5f90 HDFS-16397. Reconfig slow disk parameters for datanode (#3828)
new 5e10970 HADOOP-17765. ABFS: Use Unique File Paths in Tests. (#3153)
new 0ed0375 HADOOP-17862. ABFS: Fix unchecked cast compiler warning for
AbfsListStatusRemoteIterator (#3331)
new 36a50ba HADOOP-18075. ABFS: Fix failure caused by listFiles() in
ITestAbfsRestOperationException (#4040)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 8 +-
.../{AbfsHttpHeader.java => AbfsListResult.java} | 39 ++--
.../services/AbfsListStatusRemoteIterator.java | 71 +++
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 21 +-
.../apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 11 +-
.../ITestAbfsListStatusRemoteIterator.java | 220 +
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 12 +-
.../azurebfs/ITestAbfsRestOperationException.java | 5 +-
.../hadoop/fs/azurebfs/ITestAbfsStatistics.java| 18 +-
.../fs/azurebfs/ITestAbfsStreamStatistics.java | 4 +-
.../azurebfs/ITestAzureBlobFileSystemAppend.java | 23 +--
.../ITestAzureBlobFileSystemAuthorization.java | 6 +-
.../ITestAzureBlobFileSystemBackCompat.java| 9 +-
.../ITestAzureBlobFileSystemCheckAccess.java | 3 +-
.../fs/azurebfs/ITestAzureBlobFileSystemCopy.java | 2 +-
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 26 ++-
.../ITestAzureBlobFileSystemDelegationSAS.java | 34 ++--
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 27 +--
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 59 +++---
.../ITestAzureBlobFileSystemFileStatus.java| 28 +--
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 15 +-
.../ITestAzureBlobFileSystemListStatus.java| 31 +--
.../fs/azurebfs/ITestAzureBlobFileSystemMkDir.java | 8 +-
.../fs/azurebfs/ITestAzureBlobFileSystemOauth.java | 58 +++---
.../ITestAzureBlobFileSystemPermission.java| 3 +-
.../ITestAzureBlobFileSystemRandomRead.java| 19 +-
.../azurebfs/ITestAzureBlobFileSystemRename.java | 43 ++--
.../ITestAzureBlobFileSystemRenameUnicode.java | 2 +-
.../fs/azurebfs/ITestAzureBlobFilesystemAcl.java | 5 +-
.../fs/azurebfs/ITestCustomerProvidedKey.java | 60 +++---
.../fs/azurebfs/ITestFileSystemProperties.java | 31 +--
.../fs/azurebfs/ITestWasbAbfsCompatibility.java| 20 +-
.../azurebfs/services/ITestAbfsOutputStream.java | 26 ++-
.../fs/azurebfs/services/TestAbfsInputStream.java | 8 +-
34 files changed, 511 insertions(+), 444 deletions(-)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/{AbfsHttpHeader.java
=> AbfsListResult.java} (51%)
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-18075. ABFS: Fix failure caused by listFiles() in ITestAbfsRestOperationException (#4040)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b56af00 HADOOP-18075. ABFS: Fix failure caused by listFiles() in
ITestAbfsRestOperationException (#4040)
b56af00 is described below
commit b56af00114c5647d647703ac9b0b491933903788
Author: Steve Loughran
AuthorDate: Tue Mar 1 11:48:10 2022 +
HADOOP-18075. ABFS: Fix failure caused by listFiles() in
ITestAbfsRestOperationException (#4040)
Contributed by Sumangala Patki
---
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java| 8 ++---
.../services/AbfsListStatusRemoteIterator.java | 36 ++
.../ITestAbfsListStatusRemoteIterator.java | 34 +---
.../azurebfs/ITestAbfsRestOperationException.java | 5 +--
4 files changed, 45 insertions(+), 38 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 4786614..ae70b8d 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -1193,7 +1193,7 @@ public class AzureBlobFileSystem extends FileSystem
TracingContext tracingContext = new TracingContext(clientCorrelationId,
fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat,
listener);
AbfsListStatusRemoteIterator abfsLsItr =
- new AbfsListStatusRemoteIterator(getFileStatus(path,
tracingContext), abfsStore,
+ new AbfsListStatusRemoteIterator(path, abfsStore,
tracingContext);
return RemoteIterators.typeCastingRemoteIterator(abfsLsItr);
} else {
@@ -1368,9 +1368,9 @@ public class AzureBlobFileSystem extends FileSystem
* @throws IOException if the exception error code is not on the allowed
list.
*/
@VisibleForTesting
- static void checkException(final Path path,
- final AzureBlobFileSystemException exception,
- final AzureServiceErrorCode...
allowedErrorCodesList) throws IOException {
+ public static void checkException(final Path path,
+ final AzureBlobFileSystemException exception,
+ final AzureServiceErrorCode... allowedErrorCodesList) throws IOException
{
if (exception instanceof AbfsRestOperationException) {
AbfsRestOperationException ere = (AbfsRestOperationException) exception;
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
index ce6207b..3fecb1f 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
@@ -32,7 +32,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import
org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
public class AbfsListStatusRemoteIterator
@@ -45,7 +48,7 @@ public class AbfsListStatusRemoteIterator
private static final int MAX_QUEUE_SIZE = 10;
private static final long POLL_WAIT_TIME_IN_MS = 250;
- private final FileStatus fileStatus;
+ private final Path path;
private final ListingSupport listingSupport;
private final ArrayBlockingQueue listResultQueue;
private final TracingContext tracingContext;
@@ -55,13 +58,15 @@ public class AbfsListStatusRemoteIterator
private String continuation;
private Iterator currIterator;
- public AbfsListStatusRemoteIterator(final FileStatus fileStatus,
- final ListingSupport listingSupport, TracingContext tracingContext) {
-this.fileStatus = fileStatus;
+ public AbfsListStatusRemoteIterator(final Path path,
+ final ListingSupport listingSupport, TracingContext tracingContext)
+ throws IOException {
+this.path = path;
this.listingSupport = listingSupport;
this.tracingContext = tracingContext;
listResultQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
currIterator = Collections.emptyIterator();
+addNextBatchIteratorToQueue();
fetchBatchesAsync();
}
@@ -130,9 +135,6 @@ public class AbfsListStatusRemoteIterator
Thread.currentThread().interrupt();
LOG.error("Thread got
