This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0fd36df1d233454ab94e2f44a702cb9edbec595a Author: Viraj Jasani <vjas...@apache.org> AuthorDate: Tue Sep 20 02:26:13 2022 -0700 HADOOP-18377. hadoop-aws build to add a -prefetch profile to run all tests with prefetching (#4914) Contributed by Viraj Jasani --- hadoop-tools/hadoop-aws/pom.xml | 24 ++++++++++++++++++++++ .../src/site/markdown/tools/hadoop-aws/testing.md | 14 +++++++++++++ .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 4 ++++ 3 files changed, 42 insertions(+) diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 3be260943f1..e0de1d2458c 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -52,6 +52,9 @@ <fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit> <!-- marker retention policy --> <fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention> + + <!-- Is prefetch enabled? --> + <fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled> </properties> <profiles> @@ -126,6 +129,8 @@ <!-- Markers--> <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention> <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit> + <!-- Prefetch --> + <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled> </systemPropertyVariables> </configuration> </plugin> @@ -165,6 +170,8 @@ <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention> <test.default.timeout>${test.integration.timeout}</test.default.timeout> + <!-- Prefetch --> + <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled> </systemPropertyVariables> <!-- Some tests cannot run in parallel. Tests that cover --> <!-- access to the root directory must run in isolation --> @@ -214,6 +221,8 @@ <!-- Markers--> <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention> <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit> + <!-- Prefetch --> + <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled> </systemPropertyVariables> <!-- Do a sequential run for tests that cannot handle --> <!-- parallel execution. --> @@ -270,6 +279,8 @@ <!-- Markers--> <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention> <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit> + <!-- Prefetch --> + <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled> </systemPropertyVariables> <forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds> </configuration> @@ -331,6 +342,19 @@ </properties> </profile> + <!-- Turn on prefetching--> + <profile> + <id>prefetch</id> + <activation> + <property> + <name>prefetch</name> + </property> + </activation> + <properties> + <fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled> + </properties> + </profile> + </profiles> <build> diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index bbe1d0f8081..d2ed9ede017 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with a meaningful error ("S3 Select not supported") rather than a generic Bad Request exception. +### <a name="enabling-prefetch"></a> Enabling prefetch for all tests + +The tests are run with prefetch if the `prefetch` property is set in the +maven build. This can be combined with the scale tests as well. + +```bash +mvn verify -Dprefetch + +mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8 + +mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8 +``` + + ### Testing Requester Pays By default, the requester pays tests will look for a bucket that exists on Amazon S3 diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 384142d899a..4940eeb2e8f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -575,6 +575,10 @@ public final class S3ATestUtils { DEFAULT_DIRECTORY_MARKER_POLICY); conf.set(DIRECTORY_MARKER_POLICY, directoryRetention); + boolean prefetchEnabled = + getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT); + conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled); + return conf; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org