Author: curino Date: Thu Aug 21 21:55:57 2014 New Revision: 1619608 URL: http://svn.apache.org/r1619608 Log: Merge with trunk to pick up YARN-2436
Added: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/UnknownCipherSuiteException.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/UnknownCipherSuiteException.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandCryptoAdmin.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandCryptoAdmin.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/ - copied from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/TestHdfsCryptoStreams.java - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/TestHdfsCryptoStreams.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml - copied unchanged from r1619607, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml Propchange: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1619018-1619607 Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java Thu Aug 21 21:55:57 2014 @@ -1643,6 +1643,7 @@ public class RpcProgramNfs3 extends RpcP DirectoryListing dlisting = null; Nfs3FileAttributes postOpDirAttr = null; long dotdotFileId = 0; + HdfsFileStatus dotdotStatus = null; try { String dirFileIdPath = Nfs3Utils.getFileIdPath(handle); dirStatus = dfsClient.getFileInfo(dirFileIdPath); @@ -1678,7 +1679,7 @@ public class RpcProgramNfs3 extends RpcP if (cookie == 0) { // Get dotdot fileId String dotdotFileIdPath = dirFileIdPath + "/.."; - HdfsFileStatus dotdotStatus = dfsClient.getFileInfo(dotdotFileIdPath); + dotdotStatus = dfsClient.getFileInfo(dotdotFileIdPath); if (dotdotStatus == null) { // This should not happen @@ -1723,7 +1724,8 @@ public class RpcProgramNfs3 extends RpcP postOpDirAttr.getFileId(), ".", 0, postOpDirAttr, new FileHandle( postOpDirAttr.getFileId())); entries[1] = new READDIRPLUS3Response.EntryPlus3(dotdotFileId, "..", - dotdotFileId, postOpDirAttr, new FileHandle(dotdotFileId)); + dotdotFileId, Nfs3Utils.getNfs3FileAttrFromFileStatus(dotdotStatus, + iug), new FileHandle(dotdotFileId)); for (int i = 2; i < n + 2; i++) { long fileId = fstatus[i - 2].getFileId(); Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 21 21:55:57 2014 @@ -255,6 +255,97 @@ Trunk (Unreleased) HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. (Vinayakumar B via wheat 9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS + + HDFS-6387. HDFS CLI admin tool for creating & deleting an + encryption zone. (clamb) + + HDFS-6386. HDFS Encryption Zones (clamb) + + HDFS-6388. HDFS integration with KeyProvider. (clamb) + + HDFS-6473. Protocol and API for Encryption Zones (clamb) + + HDFS-6392. Wire crypto streams for encrypted files in + DFSClient. (clamb and yliu) + + HDFS-6476. Print out the KeyProvider after finding KP successfully on + startup. (Juan Yu via wang) + + HDFS-6391. Get the Key/IV from the NameNode for encrypted files in + DFSClient. (Charles Lamb and wang) + + HDFS-6389. Rename restrictions for encryption zones. (clamb) + + HDFS-6605. Client server negotiation of cipher suite. (wang) + + HDFS-6625. Remove the Delete Encryption Zone function (clamb) + + HDFS-6516. List of Encryption Zones should be based on inodes (clamb) + + HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao) + + HDFS-6635. Refactor encryption zone functionality into new + EncryptionZoneManager class. (wang) + + HDFS-6474. Namenode needs to get the actual keys and iv from the + KeyProvider. (wang) + + HDFS-6619. Clean up encryption-related tests. (wang) + + HDFS-6405. Test Crypto streams in HDFS. (yliu via wang) + + HDFS-6490. Fix the keyid format for generated keys in + FSNamesystem.createEncryptionZone (clamb) + + HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode. + (wang) + + HDFS-6718. Remove EncryptionZoneManager lock. (wang) + + HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang) + + HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in + EZManager#createEncryptionZone. (clamb) + + HDFS-6724. Decrypt EDEK before creating + CryptoInputStream/CryptoOutputStream. (wang) + + HDFS-6509. Create a special /.reserved/raw directory for raw access to + encrypted data. (clamb via wang) + + HDFS-6771. Require specification of an encryption key when creating + an encryption zone. (wang) + + HDFS-6730. Create a .RAW extended attribute namespace. (clamb) + + HDFS-6692. Add more HDFS encryption tests. (wang) + + HDFS-6780. Batch the encryption zones listing API. (wang) + + HDFS-6394. HDFS encryption documentation. (wang) + + HDFS-6834. Improve the configuration guidance in DFSClient when there + are no Codec classes found in configs. (umamahesh) + + HDFS-6546. Add non-superuser capability to get the encryption zone + for a specific path. (clamb) + + HDFS-6733. Creating encryption zone results in NPE when + KeyProvider is null. (clamb) + + HDFS-6785. Should not be able to create encryption zone using path + to a non-directory file. (clamb) + + HDFS-6807. Fix TestReservedRawPaths. (clamb) + + HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured + as boolean. (umamahesh) + + HDFS-6817. Fix findbugs and other warnings. (yliu) + + HDFS-6839. Fix TestCLI to expect new output. (clamb) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -405,6 +496,15 @@ Release 2.6.0 - UNRELEASED HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver. (Benoy Antony via Arpit Agarwal) + HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to + hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth) + + HDFS-6878. Change MiniDFSCluster to support StorageType configuration + for individual directories. (Arpit Agarwal) + + HDFS-6758. block writer should pass the expected block size to + DataXceiverServer. (Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) @@ -523,7 +623,12 @@ Release 2.6.0 - UNRELEASED HDFS-6868. portmap and nfs3 are documented as hadoop commands instead of hdfs (brandonli) -Release 2.5.0 - UNRELEASED + HDFS-6870. Blocks and INodes could leak for Rename with overwrite flag. (Yi + Liu via jing9) + + HDFS-6890. NFS readdirplus doesn't return dotdot attributes (brandonli) + +Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/pom.xml Thu Aug 21 21:55:57 2014 @@ -304,6 +304,7 @@ http://maven.apache.org/xsd/maven-4.0.0. <include>datatransfer.proto</include> <include>fsimage.proto</include> <include>hdfs.proto</include> + <include>encryption.proto</include> </includes> </source> <output>${project.build.directory}/generated-sources/java</output> Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Thu Aug 21 21:55:57 2014 @@ -46,6 +46,7 @@ function hadoop_usage echo " snapshotDiff diff two snapshots of a directory or diff the" echo " current directory contents with a snapshot" echo " zkfc run the ZK Failover Controller daemon" + echo " crypto configure HDFS encryption zones" echo "" echo "Most commands print help when invoked w/o parameters." } @@ -89,6 +90,9 @@ case ${COMMAND} in echo "${CLASSPATH}" exit ;; + crypto) + CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin + ;; datanode) daemon="true" # Determine if we're starting a secure datanode, and Propchange: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1619018-1619607 Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Thu Aug 21 21:55:57 2014 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs; - import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -31,6 +30,7 @@ import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CryptoCodec; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -38,6 +38,8 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; @@ -59,6 +61,7 @@ import org.apache.hadoop.util.Progressab public class Hdfs extends AbstractFileSystem { DFSClient dfs; + final CryptoCodec factory; private boolean verifyChecksum = true; static { @@ -85,6 +88,7 @@ public class Hdfs extends AbstractFileSy } this.dfs = new DFSClient(theUri, conf, getStatistics()); + this.factory = CryptoCodec.getInstance(conf); } @Override @@ -97,9 +101,12 @@ public class Hdfs extends AbstractFileSy EnumSet<CreateFlag> createFlag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws IOException { - return new HdfsDataOutputStream(dfs.primitiveCreate(getUriPath(f), - absolutePermission, createFlag, createParent, replication, blockSize, - progress, bufferSize, checksumOpt), getStatistics()); + + final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f), + absolutePermission, createFlag, createParent, replication, blockSize, + progress, bufferSize, checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics, + dfsos.getInitialLen()); } @Override @@ -308,8 +315,9 @@ public class Hdfs extends AbstractFileSy @Override public HdfsDataInputStream open(Path f, int bufferSize) throws IOException, UnresolvedLinkException { - return new DFSClient.DFSDataInputStream(dfs.open(getUriPath(f), - bufferSize, verifyChecksum)); + final DFSInputStream dfsis = dfs.open(getUriPath(f), + bufferSize, verifyChecksum); + return dfs.createWrappedInputStream(dfsis); } @Override Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java Thu Aug 21 21:55:57 2014 @@ -26,8 +26,8 @@ import org.apache.hadoop.classification. /** * XAttr is the POSIX Extended Attribute model similar to that found in * traditional Operating Systems. Extended Attributes consist of one - * or more name/value pairs associated with a file or directory. Four - * namespaces are defined: user, trusted, security and system. + * or more name/value pairs associated with a file or directory. Five + * namespaces are defined: user, trusted, security, system and raw. * 1) USER namespace attributes may be used by any user to store * arbitrary information. Access permissions in this namespace are * defined by a file directory's permission bits. For sticky directories, @@ -43,6 +43,12 @@ import org.apache.hadoop.classification. * <br> * 4) SECURITY namespace attributes are used by the fs kernel for * security features. It is not visible to users. + * <br> + * 5) RAW namespace attributes are used for internal system attributes that + * sometimes need to be exposed. Like SYSTEM namespace attributes they are + * not visible to the user except when getXAttr/getXAttrs is called on a file + * or directory in the /.reserved/raw HDFS directory hierarchy. These + * attributes can only be accessed by the superuser. * <p/> * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes"> * http://en.wikipedia.org/wiki/Extended_file_attributes</a> @@ -55,7 +61,8 @@ public class XAttr { USER, TRUSTED, SECURITY, - SYSTEM; + SYSTEM, + RAW; } private final NameSpace ns; Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Aug 21 21:55:57 2014 @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension + .EncryptedKeyVersion; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; @@ -76,6 +81,7 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; +import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -95,6 +101,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CipherSuite; +import org.apache.hadoop.crypto.CryptoCodec; +import org.apache.hadoop.crypto.CryptoInputStream; +import org.apache.hadoop.crypto.CryptoOutputStream; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CacheFlag; @@ -102,6 +113,7 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; @@ -140,6 +152,9 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -249,7 +264,11 @@ public class DFSClient implements java.i private static final DFSHedgedReadMetrics HEDGED_READ_METRIC = new DFSHedgedReadMetrics(); private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; - + private final CryptoCodec codec; + @VisibleForTesting + List<CipherSuite> cipherSuites; + @VisibleForTesting + KeyProviderCryptoExtension provider; /** * DFSClient configuration */ @@ -581,7 +600,17 @@ public class DFSClient implements java.i this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority(); this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" + DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId(); - + this.codec = CryptoCodec.getInstance(conf); + this.cipherSuites = Lists.newArrayListWithCapacity(1); + if (codec != null) { + cipherSuites.add(codec.getCipherSuite()); + } + provider = DFSUtil.createKeyProviderCryptoExtension(conf); + if (provider == null) { + LOG.info("No KeyProvider found."); + } else { + LOG.info("Found KeyProvider: " + provider.toString()); + } int numResponseToDrop = conf.getInt( DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY, DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT); @@ -1280,7 +1309,93 @@ public class DFSClient implements java.i return volumeBlockLocations; } - + + /** + * Decrypts a EDEK by consulting the KeyProvider. + */ + private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo + feInfo) throws IOException { + if (provider == null) { + throw new IOException("No KeyProvider is configured, cannot access" + + " an encrypted file"); + } + EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( + feInfo.getEzKeyVersionName(), feInfo.getIV(), + feInfo.getEncryptedDataEncryptionKey()); + try { + return provider.decryptEncryptedKey(ekv); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + + /** + * Wraps the stream in a CryptoInputStream if the underlying file is + * encrypted. + */ + public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) + throws IOException { + final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); + if (feInfo != null) { + // File is encrypted, wrap the stream in a crypto stream. + KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); + CryptoCodec codec = CryptoCodec + .getInstance(conf, feInfo.getCipherSuite()); + if (codec == null) { + throw new IOException("No configuration found for the cipher suite " + + feInfo.getCipherSuite().getConfigSuffix() + " prefixed with " + + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + + ". Please see the example configuration " + + "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE " + + "at core-default.xml for details."); + } + final CryptoInputStream cryptoIn = + new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), + feInfo.getIV()); + return new HdfsDataInputStream(cryptoIn); + } else { + // No FileEncryptionInfo so no encryption. + return new HdfsDataInputStream(dfsis); + } + } + + /** + * Wraps the stream in a CryptoOutputStream if the underlying file is + * encrypted. + */ + public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos, + FileSystem.Statistics statistics) throws IOException { + return createWrappedOutputStream(dfsos, statistics, 0); + } + + /** + * Wraps the stream in a CryptoOutputStream if the underlying file is + * encrypted. + */ + public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos, + FileSystem.Statistics statistics, long startPos) throws IOException { + final FileEncryptionInfo feInfo = dfsos.getFileEncryptionInfo(); + if (feInfo != null) { + if (codec == null) { + throw new IOException("No configuration found for the cipher suite " + + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY + " value prefixed with " + + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + + ". Please see the example configuration " + + "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE " + + "at core-default.xml for details."); + } + // File is encrypted, wrap the stream in a crypto stream. + KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); + final CryptoOutputStream cryptoOut = + new CryptoOutputStream(dfsos, codec, + decrypted.getMaterial(), feInfo.getIV(), startPos); + return new HdfsDataOutputStream(cryptoOut, statistics, startPos); + } else { + // No FileEncryptionInfo present so no encryption. + return new HdfsDataOutputStream(dfsos, statistics, startPos); + } + } + public DFSInputStream open(String src) throws IOException, UnresolvedLinkException { return open(src, dfsClientConf.ioBufferSize, true, null); @@ -1483,7 +1598,8 @@ public class DFSClient implements java.i } final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this, src, masked, flag, createParent, replication, blockSize, progress, - buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs); + buffersize, dfsClientConf.createChecksum(checksumOpt), + favoredNodeStrs, cipherSuites); beginFileLease(result.getFileId(), result); return result; } @@ -1530,7 +1646,7 @@ public class DFSClient implements java.i DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt); result = DFSOutputStream.newStreamForCreate(this, src, absPermission, flag, createParent, replication, blockSize, progress, buffersize, - checksum); + checksum, null, cipherSuites); } beginFileLease(result.getFileId(), result); return result; @@ -1608,7 +1724,7 @@ public class DFSClient implements java.i final Progressable progress, final FileSystem.Statistics statistics ) throws IOException { final DFSOutputStream out = append(src, buffersize, progress); - return new HdfsDataOutputStream(out, statistics, out.getInitialLen()); + return createWrappedOutputStream(out, statistics, out.getInitialLen()); } private DFSOutputStream append(String src, int buffersize, Progressable progress) @@ -2753,6 +2869,36 @@ public class DFSClient implements java.i } } + public void createEncryptionZone(String src, String keyName) + throws IOException { + checkOpen(); + try { + namenode.createEncryptionZone(src, keyName); + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + SafeModeException.class, + UnresolvedPathException.class); + } + } + + public EncryptionZone getEZForPath(String src) + throws IOException { + checkOpen(); + try { + final EncryptionZoneWithId ezi = namenode.getEZForPath(src); + return (ezi.getId() < 0) ? null : ezi; + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + UnresolvedPathException.class); + } + } + + public RemoteIterator<EncryptionZone> listEncryptionZones() + throws IOException { + checkOpen(); + return new EncryptionZoneIterator(namenode); + } + public void setXAttr(String src, String name, byte[] value, EnumSet<XAttrSetFlag> flag) throws IOException { checkOpen(); Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Aug 21 21:55:57 2014 @@ -567,7 +567,9 @@ public class DFSConfigKeys extends Commo public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class"; public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection"; public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class"; - + public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100; + public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses"; + // Journal-node related configs. These are read on the JN side. public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir"; public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/"; Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Thu Aug 21 21:55:57 2014 @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.UnresolvedLi import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException; @@ -92,6 +93,7 @@ implements ByteBufferReadable, CanSetDro private final boolean verifyChecksum; private LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; + private FileEncryptionInfo fileEncryptionInfo = null; private DatanodeInfo currentNode = null; private LocatedBlock currentLocatedBlock = null; private long pos = 0; @@ -301,6 +303,8 @@ implements ByteBufferReadable, CanSetDro } } + fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); + currentNode = null; return lastBlockBeingWrittenLength; } @@ -1525,6 +1529,10 @@ implements ByteBufferReadable, CanSetDro return new ReadStatistics(readStatistics); } + public synchronized FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + private synchronized void closeCurrentBlockReader() { if (blockReader == null) return; // Close the current block reader so that the new caching settings can Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Thu Aug 21 21:55:57 2014 @@ -42,10 +42,12 @@ import java.util.concurrent.atomic.Atomi import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.CanSetDropBehind; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSOutputSummer; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.fs.permission.FsPermission; @@ -153,7 +155,8 @@ public class DFSOutputStream extends FSO private boolean shouldSyncBlock = false; // force blocks to disk upon close private final AtomicReference<CachingStrategy> cachingStrategy; private boolean failPacket = false; - + private FileEncryptionInfo fileEncryptionInfo; + private static class Packet { private static final long HEART_BEAT_SEQNO = -1L; final long seqno; // sequencenumber of buffer in block @@ -1339,8 +1342,14 @@ public class DFSOutputStream extends FSO // BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage(): stage; + + // We cannot change the block length in 'block' as it counts the number + // of bytes ack'ed. + ExtendedBlock blockCopy = new ExtendedBlock(block); + blockCopy.setNumBytes(blockSize); + // send the request - new Sender(out).writeBlock(block, nodeStorageTypes[0], accessToken, + new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken, dfsClient.clientName, nodes, nodeStorageTypes, null, bcs, nodes.length, block.getNumBytes(), bytesSent, newGS, checksum, cachingStrategy.get()); @@ -1560,6 +1569,7 @@ public class DFSOutputStream extends FSO this.fileId = stat.getFileId(); this.blockSize = stat.getBlockSize(); this.blockReplication = stat.getReplication(); + this.fileEncryptionInfo = stat.getFileEncryptionInfo(); this.progress = progress; this.cachingStrategy = new AtomicReference<CachingStrategy>( dfsClient.getDefaultWriteCachingStrategy()); @@ -1600,12 +1610,13 @@ public class DFSOutputStream extends FSO static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent, short replication, long blockSize, Progressable progress, int buffersize, - DataChecksum checksum, String[] favoredNodes) throws IOException { + DataChecksum checksum, String[] favoredNodes, + List<CipherSuite> cipherSuites) throws IOException { final HdfsFileStatus stat; try { stat = dfsClient.namenode.create(src, masked, dfsClient.clientName, new EnumSetWritable<CreateFlag>(flag), createParent, replication, - blockSize); + blockSize, cipherSuites); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, @@ -1615,7 +1626,8 @@ public class DFSOutputStream extends FSO NSQuotaExceededException.class, SafeModeException.class, UnresolvedPathException.class, - SnapshotAccessControlException.class); + SnapshotAccessControlException.class, + UnknownCipherSuiteException.class); } final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat, flag, progress, checksum, favoredNodes); @@ -1623,14 +1635,6 @@ public class DFSOutputStream extends FSO return out; } - static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, - FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent, - short replication, long blockSize, Progressable progress, int buffersize, - DataChecksum checksum) throws IOException { - return newStreamForCreate(dfsClient, src, masked, flag, createParent, replication, - blockSize, progress, buffersize, checksum, null); - } - /** Construct a new output stream for append. */ private DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, @@ -1648,6 +1652,7 @@ public class DFSOutputStream extends FSO checksum.getBytesPerChecksum()); streamer = new DataStreamer(); } + this.fileEncryptionInfo = stat.getFileEncryptionInfo(); } static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src, @@ -2172,11 +2177,18 @@ public class DFSOutputStream extends FSO /** * Returns the size of a file as it was when this stream was opened */ - long getInitialLen() { + public long getInitialLen() { return initialFileSize; } /** + * @return the FileEncryptionInfo for this stream, or null if not encrypted. + */ + public FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + + /** * Returns the access token currently used by streamer, for testing only */ synchronized Token<BlockTokenIdentifier> getBlockToken() { Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Aug 21 21:55:57 2014 @@ -71,6 +71,9 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -1722,4 +1725,39 @@ public class DFSUtil { } } } + + /** + * Creates a new KeyProviderCryptoExtension by wrapping the + * KeyProvider specified in the given Configuration. + * + * @param conf Configuration specifying a single, non-transient KeyProvider. + * @return new KeyProviderCryptoExtension, or null if no provider was found. + * @throws IOException if the KeyProvider is improperly specified in + * the Configuration + */ + public static KeyProviderCryptoExtension createKeyProviderCryptoExtension( + final Configuration conf) throws IOException { + final List<KeyProvider> providers = KeyProviderFactory.getProviders(conf); + if (providers == null || providers.size() == 0) { + return null; + } + if (providers.size() > 1) { + StringBuilder builder = new StringBuilder(); + builder.append("Found multiple KeyProviders but only one is permitted ["); + String prefix = " "; + for (KeyProvider kp: providers) { + builder.append(prefix + kp.toString()); + prefix = ", "; + } + builder.append("]"); + throw new IOException(builder.toString()); + } + KeyProviderCryptoExtension provider = KeyProviderCryptoExtension + .createKeyProviderCryptoExtension(providers.get(0)); + if (provider.isTransient()) { + throw new IOException("KeyProvider " + provider.toString() + + " was found but it is a transient provider."); + } + return provider; + } } Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Thu Aug 21 21:55:57 2014 @@ -61,7 +61,6 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.client.HdfsAdmin; -import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; @@ -69,6 +68,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; @@ -291,8 +291,9 @@ public class DistributedFileSystem exten @Override public FSDataInputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataInputStream( - dfs.open(getPathName(p), bufferSize, verifyChecksum)); + final DFSInputStream dfsis = + dfs.open(getPathName(p), bufferSize, verifyChecksum); + return dfs.createWrappedInputStream(dfsis); } @Override public FSDataInputStream next(final FileSystem fs, final Path p) @@ -357,7 +358,7 @@ public class DistributedFileSystem exten : EnumSet.of(CreateFlag.CREATE), true, replication, blockSize, progress, bufferSize, null, favoredNodes); - return new HdfsDataOutputStream(out, statistics); + return dfs.createWrappedOutputStream(out, statistics); } @Override public HdfsDataOutputStream next(final FileSystem fs, final Path p) @@ -385,9 +386,10 @@ public class DistributedFileSystem exten @Override public FSDataOutputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataOutputStream(dfs.create(getPathName(p), permission, - cflags, replication, blockSize, progress, bufferSize, checksumOpt), - statistics); + final DFSOutputStream dfsos = dfs.create(getPathName(p), permission, + cflags, replication, blockSize, progress, bufferSize, + checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics); } @Override public FSDataOutputStream next(final FileSystem fs, final Path p) @@ -404,11 +406,12 @@ public class DistributedFileSystem exten short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { statistics.incrementWriteOps(1); - return new HdfsDataOutputStream(dfs.primitiveCreate( - getPathName(fixRelativePart(f)), - absolutePermission, flag, true, replication, blockSize, - progress, bufferSize, checksumOpt),statistics); - } + final DFSOutputStream dfsos = dfs.primitiveCreate( + getPathName(fixRelativePart(f)), + absolutePermission, flag, true, replication, blockSize, + progress, bufferSize, checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics); + } /** * Same as create(), except fails if parent directory doesn't already exist. @@ -428,9 +431,9 @@ public class DistributedFileSystem exten @Override public FSDataOutputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataOutputStream(dfs.create(getPathName(p), permission, - flag, false, replication, blockSize, progress, bufferSize, null), - statistics); + final DFSOutputStream dfsos = dfs.create(getPathName(p), permission, + flag, false, replication, blockSize, progress, bufferSize, null); + return dfs.createWrappedOutputStream(dfsos, statistics); } @Override @@ -1796,6 +1799,25 @@ public class DistributedFileSystem exten }.resolve(this, absF); } + /* HDFS only */ + public void createEncryptionZone(Path path, String keyName) + throws IOException { + dfs.createEncryptionZone(getPathName(path), keyName); + } + + /* HDFS only */ + public EncryptionZone getEZForPath(Path path) + throws IOException { + Preconditions.checkNotNull(path); + return dfs.getEZForPath(getPathName(path)); + } + + /* HDFS only */ + public RemoteIterator<EncryptionZone> listEncryptionZones() + throws IOException { + return dfs.listEncryptionZones(); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag) throws IOException { Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java Thu Aug 21 21:55:57 2014 @@ -49,9 +49,9 @@ public class XAttrHelper { Preconditions.checkNotNull(name, "XAttr name cannot be null."); final int prefixIndex = name.indexOf("."); - if (prefixIndex < 4) {// Prefix length is at least 4. + if (prefixIndex < 3) {// Prefix length is at least 3. throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } else if (prefixIndex == name.length() - 1) { throw new HadoopIllegalArgumentException("XAttr name cannot be empty."); } @@ -66,9 +66,11 @@ public class XAttrHelper { ns = NameSpace.SYSTEM; } else if (prefix.equals(NameSpace.SECURITY.toString().toLowerCase())) { ns = NameSpace.SECURITY; + } else if (prefix.equals(NameSpace.RAW.toString().toLowerCase())) { + ns = NameSpace.RAW; } else { throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name. substring(prefixIndex + 1)).setValue(value).build(); Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java Thu Aug 21 21:55:57 2014 @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.client; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.EnumSet; @@ -33,7 +34,9 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.hdfs.tools.DFSAdmin; /** @@ -225,4 +228,51 @@ public class HdfsAdmin { public RemoteIterator<CachePoolEntry> listCachePools() throws IOException { return dfs.listCachePools(); } + + /** + * Create an encryption zone rooted at an empty existing directory, using the + * specified encryption key. An encryption zone has an associated encryption + * key used when reading and writing files within the zone. + * + * @param path The path of the root of the encryption zone. Must refer to + * an empty, existing directory. + * @param keyName Name of key available at the KeyProvider. + * @throws IOException if there was a general IO exception + * @throws AccessControlException if the caller does not have access to path + * @throws FileNotFoundException if the path does not exist + */ + public void createEncryptionZone(Path path, String keyName) + throws IOException, AccessControlException, FileNotFoundException { + dfs.createEncryptionZone(path, keyName); + } + + /** + * Get the path of the encryption zone for a given file or directory. + * + * @param path The path to get the ez for. + * + * @return The EncryptionZone of the ez, or null if path is not in an ez. + * @throws IOException if there was a general IO exception + * @throws AccessControlException if the caller does not have access to path + * @throws FileNotFoundException if the path does not exist + */ + public EncryptionZone getEncryptionZoneForPath(Path path) + throws IOException, AccessControlException, FileNotFoundException { + return dfs.getEZForPath(path); + } + + /** + * Returns a RemoteIterator which can be used to list the encryption zones + * in HDFS. For large numbers of encryption zones, the iterator will fetch + * the list of zones in a number of small batches. + * <p/> + * Since the list is fetched in batches, it does not represent a + * consistent snapshot of the entire list of encryption zones. + * <p/> + * This method can only be called by HDFS superusers. + */ + public RemoteIterator<EncryptionZone> listEncryptionZones() + throws IOException { + return dfs.listEncryptionZones(); + } } Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java Thu Aug 21 21:55:57 2014 @@ -17,17 +17,21 @@ */ package org.apache.hadoop.hdfs.client; +import java.io.InputStream; import java.io.IOException; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import com.google.common.base.Preconditions; + /** * The Hdfs implementation of {@link FSDataInputStream}. */ @@ -38,25 +42,49 @@ public class HdfsDataInputStream extends super(in); } + public HdfsDataInputStream(CryptoInputStream in) throws IOException { + super(in); + Preconditions.checkArgument(in.getWrappedStream() instanceof DFSInputStream, + "CryptoInputStream should wrap a DFSInputStream"); + } + + private DFSInputStream getDFSInputStream() { + if (in instanceof CryptoInputStream) { + return (DFSInputStream) ((CryptoInputStream) in).getWrappedStream(); + } + return (DFSInputStream) in; + } + + /** + * Get a reference to the wrapped output stream. We always want to return the + * actual underlying InputStream, even when we're using a CryptoStream. e.g. + * in the delegated methods below. + * + * @return the underlying output stream + */ + public InputStream getWrappedStream() { + return in; + } + /** * Get the datanode from which the stream is currently reading. */ public DatanodeInfo getCurrentDatanode() { - return ((DFSInputStream) in).getCurrentDatanode(); + return getDFSInputStream().getCurrentDatanode(); } /** * Get the block containing the target position. */ public ExtendedBlock getCurrentBlock() { - return ((DFSInputStream) in).getCurrentBlock(); + return getDFSInputStream().getCurrentBlock(); } /** * Get the collection of blocks that has already been located. */ public synchronized List<LocatedBlock> getAllBlocks() throws IOException { - return ((DFSInputStream) in).getAllBlocks(); + return getDFSInputStream().getAllBlocks(); } /** @@ -66,7 +94,7 @@ public class HdfsDataInputStream extends * @return The visible length of the file. */ public long getVisibleLength() throws IOException { - return ((DFSInputStream) in).getFileLength(); + return getDFSInputStream().getFileLength(); } /** @@ -76,6 +104,6 @@ public class HdfsDataInputStream extends * bytes read through HdfsDataInputStream. */ public synchronized DFSInputStream.ReadStatistics getReadStatistics() { - return ((DFSInputStream) in).getReadStatistics(); + return getDFSInputStream().getReadStatistics(); } } Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java Thu Aug 21 21:55:57 2014 @@ -18,14 +18,18 @@ package org.apache.hadoop.hdfs.client; import java.io.IOException; +import java.io.OutputStream; import java.util.EnumSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSOutputStream; +import com.google.common.base.Preconditions; + /** * The Hdfs implementation of {@link FSDataOutputStream}. */ @@ -42,6 +46,18 @@ public class HdfsDataOutputStream extend this(out, stats, 0L); } + public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats, + long startPosition) throws IOException { + super(out, stats, startPosition); + Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream, + "CryptoOutputStream should wrap a DFSOutputStream"); + } + + public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats) + throws IOException { + this(out, stats, 0L); + } + /** * Get the actual number of replicas of the current block. * @@ -55,7 +71,11 @@ public class HdfsDataOutputStream extend * @return the number of valid replicas of the current block */ public synchronized int getCurrentBlockReplication() throws IOException { - return ((DFSOutputStream)getWrappedStream()).getCurrentBlockReplication(); + OutputStream wrappedStream = getWrappedStream(); + if (wrappedStream instanceof CryptoOutputStream) { + wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); + } + return ((DFSOutputStream) wrappedStream).getCurrentBlockReplication(); } /** @@ -67,14 +87,20 @@ public class HdfsDataOutputStream extend * @see FSDataOutputStream#hsync() */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { - ((DFSOutputStream) getWrappedStream()).hsync(syncFlags); + OutputStream wrappedStream = getWrappedStream(); + if (wrappedStream instanceof CryptoOutputStream) { + ((CryptoOutputStream) wrappedStream).flush(); + wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); + } + ((DFSOutputStream) wrappedStream).hsync(syncFlags); } public static enum SyncFlag { + /** - * When doing sync to DataNodes, also update the metadata (block - * length) in the NameNode + * When doing sync to DataNodes, also update the metadata (block length) in + * the NameNode. */ UPDATE_LENGTH; } -} \ No newline at end of file +} Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Thu Aug 21 21:55:57 2014 @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -188,7 +189,8 @@ public interface ClientProtocol { @AtMostOnce public HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable<CreateFlag> flag, - boolean createParent, short replication, long blockSize) + boolean createParent, short replication, long blockSize, + List<CipherSuite> cipherSuites) throws AccessControlException, AlreadyBeingCreatedException, DSQuotaExceededException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, @@ -1267,6 +1269,31 @@ public interface ClientProtocol { public AclStatus getAclStatus(String src) throws IOException; /** + * Create an encryption zone + */ + @AtMostOnce + public void createEncryptionZone(String src, String keyName) + throws IOException; + + /** + * Get the encryption zone for a path. + */ + @Idempotent + public EncryptionZoneWithId getEZForPath(String src) + throws IOException; + + /** + * Used to implement cursor-based batched listing of {@EncryptionZone}s. + * + * @param prevId ID of the last item in the previous batch. If there is no + * previous batch, a negative value can be used. + * @return Batch of encryption zones. + */ + @Idempotent + public BatchedEntries<EncryptionZoneWithId> listEncryptionZones( + long prevId) throws IOException; + + /** * Set xattr of a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". @@ -1307,7 +1334,6 @@ public interface ClientProtocol { * Refer to the HDFS extended attributes user documentation for details. * * @param src file or directory - * @param xAttrs xAttrs to get * @return List<XAttr> <code>XAttr</code> list * @throws IOException */ Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java Thu Aug 21 21:55:57 2014 @@ -21,6 +21,7 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -44,6 +45,8 @@ public class HdfsFileStatus { private final String owner; private final String group; private final long fileId; + + private final FileEncryptionInfo feInfo; // Used by dir, not including dot and dotdot. Always zero for a regular file. private final int childrenNum; @@ -63,11 +66,12 @@ public class HdfsFileStatus { * @param group the group of the path * @param path the local name in java UTF8 encoding the same as that in-memory * @param fileId the file id + * @param feInfo the file's encryption info */ public HdfsFileStatus(long length, boolean isdir, int block_replication, - long blocksize, long modification_time, long access_time, - FsPermission permission, String owner, String group, - byte[] symlink, byte[] path, long fileId, int childrenNum) { + long blocksize, long modification_time, long access_time, + FsPermission permission, String owner, String group, byte[] symlink, + byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -85,6 +89,7 @@ public class HdfsFileStatus { this.path = path; this.fileId = fileId; this.childrenNum = childrenNum; + this.feInfo = feInfo; } /** @@ -238,6 +243,10 @@ public class HdfsFileStatus { return fileId; } + public final FileEncryptionInfo getFileEncryptionInfo() { + return feInfo; + } + public final int getChildrenNum() { return childrenNum; } Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java Thu Aug 21 21:55:57 2014 @@ -21,6 +21,7 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -51,15 +52,16 @@ public class HdfsLocatedFileStatus exten * @param path local path name in java UTF8 format * @param fileId the file id * @param locations block locations + * @param feInfo file encryption info */ public HdfsLocatedFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, LocatedBlocks locations, - int childrenNum) { + int childrenNum, FileEncryptionInfo feInfo) { super(length, isdir, block_replication, blocksize, modification_time, - access_time, permission, owner, group, symlink, path, fileId, - childrenNum); + access_time, permission, owner, group, symlink, path, fileId, + childrenNum, feInfo); this.locations = locations; } Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java Thu Aug 21 21:55:57 2014 @@ -23,6 +23,7 @@ import java.util.Comparator; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; /** * Collection of blocks with their locations and the file length. @@ -35,22 +36,23 @@ public class LocatedBlocks { private final boolean underConstruction; private LocatedBlock lastLocatedBlock = null; private boolean isLastBlockComplete = false; + private FileEncryptionInfo fileEncryptionInfo = null; public LocatedBlocks() { fileLength = 0; blocks = null; underConstruction = false; } - - /** public Constructor */ + public LocatedBlocks(long flength, boolean isUnderConstuction, - List<LocatedBlock> blks, - LocatedBlock lastBlock, boolean isLastBlockCompleted) { + List<LocatedBlock> blks, LocatedBlock lastBlock, + boolean isLastBlockCompleted, FileEncryptionInfo feInfo) { fileLength = flength; blocks = blks; underConstruction = isUnderConstuction; this.lastLocatedBlock = lastBlock; this.isLastBlockComplete = isLastBlockCompleted; + this.fileEncryptionInfo = feInfo; } /** @@ -92,13 +94,20 @@ public class LocatedBlocks { } /** - * Return ture if file was under construction when - * this LocatedBlocks was constructed, false otherwise. + * Return true if file was under construction when this LocatedBlocks was + * constructed, false otherwise. */ public boolean isUnderConstruction() { return underConstruction; } - + + /** + * @return the FileEncryptionInfo for the LocatedBlocks + */ + public FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + /** * Find block containing specified offset. * Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java Thu Aug 21 21:55:57 2014 @@ -61,7 +61,7 @@ public class SnapshottableDirectoryStatu int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, access_time, permission, owner, group, null, localName, inodeId, - childrenNum); + childrenNum, null); this.snapshotNumber = snapshotNumber; this.snapshotQuota = snapshotQuota; this.parentFullPath = parentFullPath; Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java Thu Aug 21 21:55:57 2014 @@ -162,8 +162,10 @@ public final class DataTransferSaslUtil Configuration saslPropsResolverConf = new Configuration(conf); saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops); Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass( - DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, + HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, SaslPropertiesResolver.class, SaslPropertiesResolver.class); + resolverClass = conf.getClass(DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, + resolverClass, SaslPropertiesResolver.class); saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, resolverClass, SaslPropertiesResolver.class); SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance( Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1619608&r1=1619607&r2=1619608&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Thu Aug 21 21:55:57 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -176,6 +177,12 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; @@ -376,7 +383,8 @@ public class ClientNamenodeProtocolServe HdfsFileStatus result = server.create(req.getSrc(), PBHelper.convert(req.getMasked()), req.getClientName(), PBHelper.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(), - (short) req.getReplication(), req.getBlockSize()); + (short) req.getReplication(), req.getBlockSize(), + PBHelper.convertCipherSuiteProtos(req.getCipherSuitesList())); if (result != null) { return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result)) @@ -1301,6 +1309,52 @@ public class ClientNamenodeProtocolServe } @Override + public CreateEncryptionZoneResponseProto createEncryptionZone( + RpcController controller, CreateEncryptionZoneRequestProto req) + throws ServiceException { + try { + server.createEncryptionZone(req.getSrc(), req.getKeyName()); + return CreateEncryptionZoneResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetEZForPathResponseProto getEZForPath( + RpcController controller, GetEZForPathRequestProto req) + throws ServiceException { + try { + GetEZForPathResponseProto.Builder builder = + GetEZForPathResponseProto.newBuilder(); + final EncryptionZoneWithId ret = server.getEZForPath(req.getSrc()); + builder.setZone(PBHelper.convert(ret)); + return builder.build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ListEncryptionZonesResponseProto listEncryptionZones( + RpcController controller, ListEncryptionZonesRequestProto req) + throws ServiceException { + try { + BatchedEntries<EncryptionZoneWithId> entries = server + .listEncryptionZones(req.getId()); + ListEncryptionZonesResponseProto.Builder builder = + ListEncryptionZonesResponseProto.newBuilder(); + builder.setHasMore(entries.hasMore()); + for (int i=0; i<entries.size(); i++) { + builder.addZones(PBHelper.convert(entries.get(i))); + } + return builder.build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override public SetXAttrResponseProto setXAttr(RpcController controller, SetXAttrRequestProto req) throws ServiceException { try {