This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch branch-3.4.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4.0 by this push:
new c8ee14a2eb2f HADOOP-19045. CreateSession Timeout - followup (#6532)
c8ee14a2eb2f is described below
commit c8ee14a2eb2fd1ea435945b6740b77c73c132f8a
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Wed Feb 7 12:07:54 2024 +0000
HADOOP-19045. CreateSession Timeout - followup (#6532)
This is a followup to PR:
HADOOP-19045. S3A: Validate CreateSession Timeout Propagation (#6470)
Remove all declarations of fs.s3a.connection.request.timeout
in
- hadoop-common/src/main/resources/core-default.xml
- hadoop-aws/src/test/resources/core-site.xml
New test in TestAwsClientConfig to verify that the value
defined in fs.s3a.Constants class is used.
This is brittle to someone overriding it in their test setups,
but as this test is intended to verify that the option is not
explicitly set, there's no workaround.
Contributed by Steve Loughran
---
.../hadoop-common/src/main/resources/core-default.xml | 14 --------------
.../src/site/markdown/tools/hadoop-aws/performance.md | 2 +-
.../markdown/tools/hadoop-aws/troubleshooting_s3a.md | 2 +-
.../apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java | 16 ++++++++++++++++
hadoop-tools/hadoop-aws/src/test/resources/core-site.xml | 5 -----
5 files changed, 18 insertions(+), 21 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 29ec06db6598..00533e362b99 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2097,20 +2097,6 @@
</description>
</property>
-<property>
- <name>fs.s3a.connection.request.timeout</name>
- <value>0s</value>
- <description>
- Time out on HTTP requests to the AWS service; 0 means no timeout.
-
- Important: this is the maximum duration of any AWS service call,
- including upload and copy operations. If non-zero, it must be larger
- than the time to upload multi-megabyte blocks to S3 from the client,
- and to rename many-GB files. Use with care.
-
- </description>
-</property>
-
<property>
<name>fs.s3a.etag.checksum.enabled</name>
<value>false</value>
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
index 4c03cca17161..44c8086d4f02 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
@@ -231,7 +231,7 @@ The choice is yours. Generally recovery is better, but
sometimes fail-fast is mo
| `fs.s3a.connection.acquisition.timeout` | `60s` | `*` | Timeout for
waiting for a connection from the pool. |
| `fs.s3a.connection.establish.timeout` | `30s` | | Time to establish
the TCP/TLS connection |
| `fs.s3a.connection.idle.time` | `60s` | `*` | Maximum time for
idle HTTP connections in the pool |
-| `fs.s3a.connection.request.timeout` | `0` | | If greater than
zero, maximum duration of any request |
+| `fs.s3a.connection.request.timeout` | `60s` | | If greater than
zero, maximum time for a response |
| `fs.s3a.connection.timeout` | `200s` | | Timeout for socket
problems on a TCP channel |
| `fs.s3a.connection.ttl` | `5m` | | Lifetime of HTTP
connections from the pool |
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index c1b499e3da95..e53e4a002265 100644
---
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -1510,7 +1510,7 @@ It is possible to configure a global timeout for AWS
service calls using followi
```xml
<property>
<name>fs.s3a.connection.request.timeout</name>
- <value>0</value>
+ <value>5m</value>
<description>
Time out on HTTP requests to the AWS service; 0 means no timeout.
Measured in seconds; the usual time suffixes are all supported
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
index 2a9e05755ce6..eacff90ea4c8 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java
@@ -40,6 +40,7 @@ import static
org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_KEEPALIVE;
import static
org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL_DURATION;
import static
org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT_DURATION;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS;
+import static
org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT_DURATION;
import static
org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT_DURATION;
import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
@@ -175,6 +176,21 @@ public class TestAwsClientConfig extends
AbstractHadoopTestBase {
.describedAs("%s in %s", REQUEST_TIMEOUT, settings)
.isEqualTo(Duration.ofHours(1));
}
+ /**
+ * Verify that the timeout from {@link
org.apache.hadoop.fs.s3a.Constants#DEFAULT_REQUEST_TIMEOUT_DURATION}
+ * makes it all the way through and that nothing in in core-default or
core-site is setting it.
+ * This test will fail if someone does set it in core-site.xml
+ */
+ @Test
+ public void testCreateApiConnectionSettingsDefault() {
+ final Configuration conf = new Configuration();
+ Assertions.assertThat(conf.get(REQUEST_TIMEOUT))
+ .describedAs("Request timeout %s", REQUEST_TIMEOUT)
+ .isNull();
+
+ assertDuration(REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT_DURATION,
+ createApiConnectionSettings(conf).getApiCallTimeout());
+ }
/**
* Set a list of keys to the same value.
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 70b87ee27540..30e54ca0b87d 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -201,11 +201,6 @@
<value>true</value>
</property>
- <property>
- <name>fs.s3a.connection.request.timeout</name>
- <value>10s</value>
- </property>
-
<property>
<name>fs.s3a.attempts.maximum</name>
<value>1</value>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
