This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new aa6a9ca HADOOP-17596. ABFS: Change default Readahead Queue Depth from
num(processors) to const (#3106)
aa6a9ca is described below
commit aa6a9cac7254d87fca3d7b74c160f879792fdf74
Author: sumangala-patki <70206833+sumangala-pa...@users.noreply.github.com>
AuthorDate: Sat Jul 10 15:09:59 2021 +0530
HADOOP-17596. ABFS: Change default Readahead Queue Depth from
num(processors) to const (#3106)
* HADOOP-17596. ABFS: Change default Readahead Queue Depth from
num(processors) to const (#2795)
. Contributed by Sumangala Patki.
(cherry picked from commit 76d92eb2a22c71b5fcde88a9b4d2faec81a1cb9f)
---
.../fs/azurebfs/constants/FileSystemConfigurations.java | 2 +-
hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 2 +-
.../hadoop/fs/azurebfs/services/TestAbfsInputStream.java | 16 ++++++++++++++++
3 files changed, 18 insertions(+), 2 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 4194d21..6ef3ca4 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -89,7 +89,7 @@ public final class FileSystemConfigurations {
public static final int MIN_LEASE_DURATION = 15;
public static final int MAX_LEASE_DURATION = 60;
- public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = -1;
+ public static final int DEFAULT_READ_AHEAD_QUEUE_DEPTH = 2;
public static final boolean DEFAULT_ENABLE_FLUSH = true;
public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true;
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index 6be5952..8724b97 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -803,7 +803,7 @@ pattern is detected.
`fs.azure.readaheadqueue.depth`: Sets the readahead queue depth in
AbfsInputStream. In case the set value is negative the read ahead queue depth
will be set as Runtime.getRuntime().availableProcessors(). By default the value
-will be -1. To disable readaheads, set this value to 0. If your workload is
+will be 2. To disable readaheads, set this value to 0. If your workload is
doing only random reads (non-sequential) or you are seeing throttling, you
may try setting this value to 0.
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
index cbf3d6a..3da004b 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
@@ -28,6 +28,7 @@ import org.assertj.core.api.Assertions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -48,6 +49,7 @@ import static org.mockito.Mockito.when;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH;
/**
* Unit test AbfsInputStream.
@@ -569,6 +571,20 @@ public class TestAbfsInputStream extends
testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
}
+ @Test
+ public void testDefaultReadaheadQueueDepth() throws Exception {
+ Configuration config = getRawConfiguration();
+ config.unset(FS_AZURE_READ_AHEAD_QUEUE_DEPTH);
+ AzureBlobFileSystem fs = getFileSystem(config);
+ Path testFile = new Path("/testFile");
+ fs.create(testFile);
+ FSDataInputStream in = fs.open(testFile);
+ Assertions.assertThat(
+ ((AbfsInputStream) in.getWrappedStream()).getReadAheadQueueDepth())
+ .describedAs("readahead queue depth should be set to default value 2")
+ .isEqualTo(2);
+ }
+
private void testReadAheads(AbfsInputStream inputStream,
int readRequestSize,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
