Author: atm Date: Sat May 3 00:26:23 2014 New Revision: 1592134 URL: http://svn.apache.org/r1592134 Log: HADOOP-10568. Add s3 server-side encryption. Contributed by David S. Wang.
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1592134&r1=1592133&r2=1592134&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Sat May 3 00:26:23 2014 @@ -111,6 +111,8 @@ Release 2.5.0 - UNRELEASED HADOOP-10562. Namenode exits on exception without printing stack trace in AbstractDelegationTokenSecretManager. (Arpit Agarwal) + HADOOP-10568. Add s3 server-side encryption. (David S. Wang via atm) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java?rev=1592134&r1=1592133&r2=1592134&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java Sat May 3 00:26:23 2014 @@ -63,6 +63,8 @@ class Jets3tNativeFileSystemStore implem private boolean multipartEnabled; private long multipartCopyBlockSize; static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024; + + private String serverSideEncryptionAlgorithm; public static final Log LOG = LogFactory.getLog(Jets3tNativeFileSystemStore.class); @@ -87,6 +89,7 @@ class Jets3tNativeFileSystemStore implem multipartCopyBlockSize = Math.min( conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE), MAX_PART_SIZE); + serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm"); bucket = new S3Bucket(uri.getHost()); } @@ -107,6 +110,7 @@ class Jets3tNativeFileSystemStore implem object.setDataInputStream(in); object.setContentType("binary/octet-stream"); object.setContentLength(file.length()); + object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); if (md5Hash != null) { object.setMd5Hash(md5Hash); } @@ -130,6 +134,7 @@ class Jets3tNativeFileSystemStore implem object.setDataInputFile(file); object.setContentType("binary/octet-stream"); object.setContentLength(file.length()); + object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); if (md5Hash != null) { object.setMd5Hash(md5Hash); } @@ -156,6 +161,7 @@ class Jets3tNativeFileSystemStore implem object.setDataInputStream(new ByteArrayInputStream(new byte[0])); object.setContentType("binary/octet-stream"); object.setContentLength(0); + object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.putObject(bucket, object); } catch (S3ServiceException e) { handleS3ServiceException(e); @@ -317,8 +323,11 @@ class Jets3tNativeFileSystemStore implem return; } } + + S3Object dstObject = new S3Object(dstKey); + dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), - new S3Object(dstKey), false); + dstObject, false); } catch (ServiceException e) { handleServiceException(srcKey, e); } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1592134&r1=1592133&r2=1592134&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Sat May 3 00:26:23 2014 @@ -568,6 +568,14 @@ </property> <property> + <name>fs.s3n.server-side-encryption-algorithm</name> + <value></value> + <description>Specify a server-side encryption algorithm for S3. + The default is NULL, and the only other currently allowable value is AES256. + </description> +</property> + +<property> <name>io.seqfile.compress.blocksize</name> <value>1000000</value> <description>The minimum block size for compression in block compressed