HADOOP-13122 Customize User-Agent header sent in HTTP requests by S3A. Chris Nauroth via stevel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/def2a6d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/def2a6d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/def2a6d3 Branch: refs/heads/HDFS-1312 Commit: def2a6d3856452d5c804f04e5bf485541a3bc53a Parents: b7ac852 Author: Steve Loughran <ste...@apache.org> Authored: Thu May 12 13:56:48 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu May 12 13:57:35 2016 +0100 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 15 +++++ .../org/apache/hadoop/fs/s3a/Constants.java | 2 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 23 +++++++ .../src/site/markdown/tools/hadoop-aws/index.md | 15 +++++ .../hadoop/fs/s3a/TestS3AConfiguration.java | 71 ++++++++++++++++---- 5 files changed, 113 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/def2a6d3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3d1c249..bd061c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -946,6 +946,21 @@ </property> <property> + <name>fs.s3a.user.agent.prefix</name> + <value></value> + <description> + Sets a custom value that will be prepended to the User-Agent header sent in + HTTP requests to the S3 back-end by S3AFileSystem. The User-Agent header + always includes the Hadoop version number followed by a string generated by + the AWS SDK. An example is "User-Agent: Hadoop 2.8.0, aws-sdk-java/1.10.6". + If this optional property is set, then its value is prepended to create a + customized User-Agent. For example, if this configuration property was set + to "MyApp", then an example of the resulting User-Agent would be + "User-Agent: MyApp, Hadoop 2.8.0, aws-sdk-java/1.10.6". + </description> +</property> + +<property> <name>fs.s3a.impl</name> <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> <description>The implementation class of the S3A Filesystem</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/def2a6d3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 9d79623..bf3f85f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -122,4 +122,6 @@ public class Constants { public static final String FS_S3A = "s3a"; public static final int S3A_DEFAULT_PORT = -1; + + public static final String USER_AGENT_PREFIX = "fs.s3a.user.agent.prefix"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/def2a6d3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 8739798..287474c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -69,6 +69,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.VersionInfo; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -139,6 +140,8 @@ public class S3AFileSystem extends FileSystem { initProxySupport(conf, awsConf, secureConnections); + initUserAgent(conf, awsConf); + initAmazonS3Client(conf, credentials, awsConf); maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); @@ -231,6 +234,26 @@ public class S3AFileSystem extends FileSystem { } } + /** + * Initializes the User-Agent header to send in HTTP requests to the S3 + * back-end. We always include the Hadoop version number. The user also may + * set an optional custom prefix to put in front of the Hadoop version number. + * The AWS SDK interally appends its own information, which seems to include + * the AWS SDK version, OS and JVM version. + * + * @param conf Hadoop configuration + * @param awsConf AWS SDK configuration + */ + private void initUserAgent(Configuration conf, ClientConfiguration awsConf) { + String userAgent = "Hadoop " + VersionInfo.getVersion(); + String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, ""); + if (!userAgentPrefix.isEmpty()) { + userAgent = userAgentPrefix + ", " + userAgent; + } + LOG.debug("Using User-Agent: {}", userAgent); + awsConf.setUserAgent(userAgent); + } + private void initAmazonS3Client(Configuration conf, AWSCredentialsProviderChain credentials, ClientConfiguration awsConf) throws IllegalArgumentException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/def2a6d3/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 9e026d9..e6fbe89 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -408,6 +408,21 @@ this capability. </property> <property> + <name>fs.s3a.user.agent.prefix</name> + <value></value> + <description> + Sets a custom value that will be prepended to the User-Agent header sent in + HTTP requests to the S3 back-end by S3AFileSystem. The User-Agent header + always includes the Hadoop version number followed by a string generated by + the AWS SDK. An example is "User-Agent: Hadoop 2.8.0, aws-sdk-java/1.10.6". + If this optional property is set, then its value is prepended to create a + customized User-Agent. For example, if this configuration property was set + to "MyApp", then an example of the resulting User-Agent would be + "User-Agent: MyApp, Hadoop 2.8.0, aws-sdk-java/1.10.6". + </description> + </property> + + <property> <name>fs.s3a.impl</name> <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> <description>The implementation class of the S3A Filesystem</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/def2a6d3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java index 4a0bfbb..7928ea9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java @@ -18,11 +18,13 @@ package org.apache.hadoop.fs.s3a; +import com.amazonaws.ClientConfiguration; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.model.AmazonS3Exception; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.reflect.FieldUtils; import com.amazonaws.AmazonClientException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -46,6 +48,7 @@ import java.lang.reflect.Field; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.util.VersionInfo; import org.apache.http.HttpStatus; import org.junit.rules.TemporaryFolder; @@ -370,10 +373,13 @@ public class TestS3AConfiguration { try { fs = S3ATestUtils.createTestFileSystem(conf); - final Object object = getClientOptionsField(fs.getAmazonS3Client(), "clientOptions"); - assertNotNull(object); - assertTrue("Unexpected type found for clientOptions!", object instanceof S3ClientOptions); - assertTrue("Expected to find path style access to be switched on!", ((S3ClientOptions) object).isPathStyleAccess()); + assertNotNull(fs); + AmazonS3Client s3 = fs.getAmazonS3Client(); + assertNotNull(s3); + S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class, + "clientOptions"); + assertTrue("Expected to find path style access to be switched on!", + clientOptions.isPathStyleAccess()); byte[] file = ContractTestUtils.toAsciiByteArray("test file"); ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true); } catch (final AmazonS3Exception e) { @@ -385,14 +391,53 @@ public class TestS3AConfiguration { } } - private Object getClientOptionsField(AmazonS3Client s3client, String field) - throws NoSuchFieldException, IllegalAccessException { - final Field clientOptionsProps = s3client.getClass().getDeclaredField(field); - assertNotNull(clientOptionsProps); - if (!clientOptionsProps.isAccessible()) { - clientOptionsProps.setAccessible(true); - } - final Object object = clientOptionsProps.get(s3client); - return object; + @Test + public void testDefaultUserAgent() throws Exception { + conf = new Configuration(); + fs = S3ATestUtils.createTestFileSystem(conf); + assertNotNull(fs); + AmazonS3Client s3 = fs.getAmazonS3Client(); + assertNotNull(s3); + ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, + "clientConfiguration"); + assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent()); + } + + @Test + public void testCustomUserAgent() throws Exception { + conf = new Configuration(); + conf.set(Constants.USER_AGENT_PREFIX, "MyApp"); + fs = S3ATestUtils.createTestFileSystem(conf); + assertNotNull(fs); + AmazonS3Client s3 = fs.getAmazonS3Client(); + assertNotNull(s3); + ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, + "clientConfiguration"); + assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(), + awsConf.getUserAgent()); + } + + /** + * Reads and returns a field from an object using reflection. If the field + * cannot be found, is null, or is not the expected type, then this method + * fails the test. + * + * @param target object to read + * @param fieldType type of field to read, which will also be the return type + * @param fieldName name of field to read + * @return field that was read + * @throws IllegalAccessException if access not allowed + */ + private static <T> T getField(Object target, Class<T> fieldType, + String fieldName) throws IllegalAccessException { + Object obj = FieldUtils.readField(target, fieldName, true); + assertNotNull(String.format( + "Could not read field named %s in object with class %s.", fieldName, + target.getClass().getName()), obj); + assertTrue(String.format( + "Unexpected type found for field named %s, expected %s, actual %s.", + fieldName, fieldType.getName(), obj.getClass().getName()), + fieldType.isAssignableFrom(obj.getClass())); + return fieldType.cast(obj); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
