This is an automated email from the ASF dual-hosted git repository.
anujmodi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fc42da70cea HADOOP-19357: [ABFS] Optimizations for Retry Handling and
Client Side Throttling (#7216)
fc42da70cea is described below
commit fc42da70cea536e67997636d8fcb7833932a5d98
Author: manika137 <[email protected]>
AuthorDate: Tue Dec 17 02:02:00 2024 -0800
HADOOP-19357: [ABFS] Optimizations for Retry Handling and Client Side
Throttling (#7216)
Default for following configs are changed:
Client-side throttling (CST): Off
Client Backoff - 500ms (reduced from 3sec)
Max Backoff - 25s (reduced from 30sec)
Min Backoff - 500ms (reduced from 3sec)
Contributed by Manika Joshi (@manika137)
---
.../azurebfs/constants/FileSystemConfigurations.java | 8 ++++----
.../azurebfs/services/ITestExponentialRetryPolicy.java | 18 ++++++++++++++++++
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index aeb31969666..6ada0e8bcca 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -39,11 +39,11 @@ public final class FileSystemConfigurations {
private static final int SIXTY_SECONDS = 60_000;
// Retry parameter defaults.
- public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3_000; // 3s
- public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30_000; // 30s
+ public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 500; // 500ms
+ public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 25_000; // 25s
public static final boolean
DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED = true;
public static final int DEFAULT_STATIC_RETRY_INTERVAL = 1_000; // 1s
- public static final int DEFAULT_BACKOFF_INTERVAL = 3_000; // 3s
+ public static final int DEFAULT_BACKOFF_INTERVAL = 500; // 500ms
public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3;
@@ -108,7 +108,7 @@ public final class FileSystemConfigurations {
public static final boolean DEFAULT_ENABLE_FLUSH = true;
public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true;
- public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true;
+ public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = false;
public static final int DEFAULT_METRIC_IDLE_TIMEOUT_MS = 60_000;
public static final int DEFAULT_METRIC_ANALYSIS_TIMEOUT_MS = 60_000;
public static final boolean
DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = true;
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
index 9ead6d19638..1d289eabfa9 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
@@ -96,6 +96,24 @@ public class ITestExponentialRetryPolicy extends
AbstractAbfsIntegrationTest {
testMaxIOConfig(abfsConfig);
}
+ @Test
+ public void testClientSideThrottlingConfigs() throws Exception {
+ final Configuration configuration = new Configuration();
+ configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, true);
+ AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+ DUMMY_ACCOUNT_NAME);
+ Assertions.assertThat(abfsConfiguration.isAutoThrottlingEnabled())
+ .describedAs("Client-side throttling enabled by configuration key")
+ .isTrue();
+
+ configuration.unset(FS_AZURE_ENABLE_AUTOTHROTTLING);
+ AbfsConfiguration abfsConfiguration2 = new AbfsConfiguration(configuration,
+ DUMMY_ACCOUNT_NAME);
+ Assertions.assertThat(abfsConfiguration2.isAutoThrottlingEnabled())
+ .describedAs("Client-side throttling should be disabled by
default")
+ .isFalse();
+ }
+
@Test
public void testThrottlingIntercept() throws Exception {
AzureBlobFileSystem fs = getFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
