This is an automated email from the ASF dual-hosted git repository.
anujmodi pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 23b0559dabc HADOOP-19357: [ABFS] Optimizations for Retry Handling and
Client Side Throttling (#7216) (#7234)
23b0559dabc is described below
commit 23b0559dabc805085e4c9d744cc068beed1888ce
Author: manika137 <[email protected]>
AuthorDate: Mon Dec 23 01:15:06 2024 -0800
HADOOP-19357: [ABFS] Optimizations for Retry Handling and Client Side
Throttling (#7216) (#7234)
Default for following configs are changed:
Client-side throttling (CST): Off
Client Backoff - 500ms (reduced from 3sec)
Max Backoff - 25s (reduced from 30sec)
Min Backoff - 500ms (reduced from 3sec)
Contributed by Manika Joshi (@manika137)
---
.../azurebfs/constants/FileSystemConfigurations.java | 10 ++++++----
.../azurebfs/services/ITestExponentialRetryPolicy.java | 18 ++++++++++++++++++
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index cef6994f8d2..16c7d18890f 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -32,17 +32,18 @@ import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_ST
public final class FileSystemConfigurations {
public static final String DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "";
+ public static final boolean DEFAULT_FS_AZURE_ENABLE_DFSTOBLOB_FALLBACK =
false;
public static final boolean
DEFAULT_FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED = true;
public static final String USER_HOME_DIRECTORY_PREFIX = "/user";
private static final int SIXTY_SECONDS = 60_000;
// Retry parameter defaults.
- public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3_000; // 3s
- public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30_000; // 30s
+ public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 500; // 500ms
+ public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 25_000; // 25s
public static final boolean
DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED = true;
public static final int DEFAULT_STATIC_RETRY_INTERVAL = 1_000; // 1s
- public static final int DEFAULT_BACKOFF_INTERVAL = 3_000; // 3s
+ public static final int DEFAULT_BACKOFF_INTERVAL = 500; // 500ms
public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3;
@@ -107,7 +108,7 @@ public final class FileSystemConfigurations {
public static final boolean DEFAULT_ENABLE_FLUSH = true;
public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true;
- public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true;
+ public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = false;
public static final boolean
DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = true;
public static final int DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS = 60_000;
public static final int DEFAULT_ANALYSIS_PERIOD_MS = 10_000;
@@ -173,5 +174,6 @@ public final class FileSystemConfigurations {
public static final long DEFAULT_HTTP_CLIENT_CONN_MAX_IDLE_TIME = 5_000L;
public static final int DEFAULT_HTTP_CLIENT_CONN_MAX_CACHED_CONNECTIONS = 5;
+
private FileSystemConfigurations() {}
}
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
index 9ead6d19638..1d289eabfa9 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
@@ -96,6 +96,24 @@ public class ITestExponentialRetryPolicy extends
AbstractAbfsIntegrationTest {
testMaxIOConfig(abfsConfig);
}
+ @Test
+ public void testClientSideThrottlingConfigs() throws Exception {
+ final Configuration configuration = new Configuration();
+ configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, true);
+ AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+ DUMMY_ACCOUNT_NAME);
+ Assertions.assertThat(abfsConfiguration.isAutoThrottlingEnabled())
+ .describedAs("Client-side throttling enabled by configuration key")
+ .isTrue();
+
+ configuration.unset(FS_AZURE_ENABLE_AUTOTHROTTLING);
+ AbfsConfiguration abfsConfiguration2 = new AbfsConfiguration(configuration,
+ DUMMY_ACCOUNT_NAME);
+ Assertions.assertThat(abfsConfiguration2.isAutoThrottlingEnabled())
+ .describedAs("Client-side throttling should be disabled by
default")
+ .isFalse();
+ }
+
@Test
public void testThrottlingIntercept() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
