This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch feature-HADOOP-18028-s3a-prefetch-branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 5aa200e8ada2564d2711b433924458fe75a7275e
Author: Viraj Jasani <vjas...@apache.org>
AuthorDate: Tue Sep 20 02:26:13 2022 -0700
HADOOP-18377. hadoop-aws build to add a -prefetch profile to run all tests
with prefetching (#4914)
Contributed by Viraj Jasani
---
hadoop-tools/hadoop-aws/pom.xml | 24 ++++++++++++++++++++++
.../src/site/markdown/tools/hadoop-aws/testing.md | 14 +++++++++++++
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 4 ++++
3 files changed, 42 insertions(+)
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 3be260943f1..e0de1d2458c 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -52,6 +52,9 @@
<fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit>
<!-- marker retention policy -->
<fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention>
+
+ <!-- Is prefetch enabled? -->
+ <fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled>
</properties>
<profiles>
@@ -126,6 +129,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+ <!-- Prefetch -->
+
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -165,6 +170,8 @@
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ <!-- Prefetch -->
+
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<!-- Some tests cannot run in parallel. Tests that cover -->
<!-- access to the root directory must run in isolation -->
@@ -214,6 +221,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+ <!-- Prefetch -->
+
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<!-- Do a sequential run for tests that cannot handle -->
<!-- parallel execution. -->
@@ -270,6 +279,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+ <!-- Prefetch -->
+
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
</configuration>
@@ -331,6 +342,19 @@
</properties>
</profile>
+ <!-- Turn on prefetching-->
+ <profile>
+ <id>prefetch</id>
+ <activation>
+ <property>
+ <name>prefetch</name>
+ </property>
+ </activation>
+ <properties>
+ <fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled>
+ </properties>
+ </profile>
+
</profiles>
<build>
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index bbe1d0f8081..d2ed9ede017 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select
fails fast with
a meaningful error ("S3 Select not supported") rather than a generic Bad
Request
exception.
+### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
+
+The tests are run with prefetch if the `prefetch` property is set in the
+maven build. This can be combined with the scale tests as well.
+
+```bash
+mvn verify -Dprefetch
+
+mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
+
+mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
+```
+
+
### Testing Requester Pays
By default, the requester pays tests will look for a bucket that exists on
Amazon S3
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 384142d899a..4940eeb2e8f 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -575,6 +575,10 @@ public final class S3ATestUtils {
DEFAULT_DIRECTORY_MARKER_POLICY);
conf.set(DIRECTORY_MARKER_POLICY, directoryRetention);
+ boolean prefetchEnabled =
+ getTestPropertyBool(conf, PREFETCH_ENABLED_KEY,
PREFETCH_ENABLED_DEFAULT);
+ conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled);
+
return conf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
