This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new f439801484f HADOOP-18246. Reduce lower limit on fs.s3a.prefetch.block.size to 1 byte. (#5120) f439801484f is described below commit f439801484fdf07329462d7c93f861167a476dfc Author: Ankit Saurabh <112006204+saura...@users.noreply.github.com> AuthorDate: Thu Feb 2 18:45:21 2023 +0000 HADOOP-18246. Reduce lower limit on fs.s3a.prefetch.block.size to 1 byte. (#5120) The minimum value of fs.s3a.prefetch.block.size is now 1 Contributed by Ankit Saurabh --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +-- hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 1 + .../hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 155a91a8af1..394452e8fe6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -543,8 +543,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, DEFAULT_MULTIPART_UPLOAD_ENABLED); this.prefetchEnabled = conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT); long prefetchBlockSizeLong = - longBytesOption(conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE, - PREFETCH_BLOCK_DEFAULT_SIZE); + longBytesOption(conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE, 1); if (prefetchBlockSizeLong > (long) Integer.MAX_VALUE) { throw new IOException("S3A prefetch block size exceeds int limit"); } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 5d41f8c7427..884cc781df9 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1107,6 +1107,7 @@ options are covered in [Testing](./testing.md). <value>8MB</value> <description> The size of a single prefetched block of data. + Decreasing this will increase the number of prefetches required, and may negatively impact performance. </description> </property> diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md index e966c2dce4c..8bb85008e36 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md @@ -43,6 +43,10 @@ Multiple blocks may be read in parallel. |`fs.s3a.prefetch.block.size` |Size of a block |`8M` | |`fs.s3a.prefetch.block.count` |Number of blocks to prefetch |`8` | +The default size of a block is 8MB, and the minimum allowed block size is 1 byte. +Decreasing block size will increase the number of blocks to be read for a file. +A smaller block size may negatively impact performance as the number of prefetches required will increase. + ### Key Components `S3PrefetchingInputStream` - When prefetching is enabled, S3AFileSystem will return an instance of --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org