Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Thu Aug 21 05:22:10 2014 @@ -42,10 +42,12 @@ import java.util.concurrent.atomic.Atomi import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.CanSetDropBehind; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSOutputSummer; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.fs.permission.FsPermission; @@ -153,7 +155,8 @@ public class DFSOutputStream extends FSO private boolean shouldSyncBlock = false; // force blocks to disk upon close private final AtomicReference<CachingStrategy> cachingStrategy; private boolean failPacket = false; - + private FileEncryptionInfo fileEncryptionInfo; + private static class Packet { private static final long HEART_BEAT_SEQNO = -1L; final long seqno; // sequencenumber of buffer in block @@ -1339,8 +1342,14 @@ public class DFSOutputStream extends FSO // BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage(): stage; + + // We cannot change the block length in 'block' as it counts the number + // of bytes ack'ed. + ExtendedBlock blockCopy = new ExtendedBlock(block); + blockCopy.setNumBytes(blockSize); + // send the request - new Sender(out).writeBlock(block, nodeStorageTypes[0], accessToken, + new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken, dfsClient.clientName, nodes, nodeStorageTypes, null, bcs, nodes.length, block.getNumBytes(), bytesSent, newGS, checksum, cachingStrategy.get()); @@ -1560,6 +1569,7 @@ public class DFSOutputStream extends FSO this.fileId = stat.getFileId(); this.blockSize = stat.getBlockSize(); this.blockReplication = stat.getReplication(); + this.fileEncryptionInfo = stat.getFileEncryptionInfo(); this.progress = progress; this.cachingStrategy = new AtomicReference<CachingStrategy>( dfsClient.getDefaultWriteCachingStrategy()); @@ -1600,12 +1610,13 @@ public class DFSOutputStream extends FSO static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent, short replication, long blockSize, Progressable progress, int buffersize, - DataChecksum checksum, String[] favoredNodes) throws IOException { + DataChecksum checksum, String[] favoredNodes, + List<CipherSuite> cipherSuites) throws IOException { final HdfsFileStatus stat; try { stat = dfsClient.namenode.create(src, masked, dfsClient.clientName, new EnumSetWritable<CreateFlag>(flag), createParent, replication, - blockSize); + blockSize, cipherSuites); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, @@ -1615,7 +1626,8 @@ public class DFSOutputStream extends FSO NSQuotaExceededException.class, SafeModeException.class, UnresolvedPathException.class, - SnapshotAccessControlException.class); + SnapshotAccessControlException.class, + UnknownCipherSuiteException.class); } final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat, flag, progress, checksum, favoredNodes); @@ -1623,14 +1635,6 @@ public class DFSOutputStream extends FSO return out; } - static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, - FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent, - short replication, long blockSize, Progressable progress, int buffersize, - DataChecksum checksum) throws IOException { - return newStreamForCreate(dfsClient, src, masked, flag, createParent, replication, - blockSize, progress, buffersize, checksum, null); - } - /** Construct a new output stream for append. */ private DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, @@ -1648,6 +1652,7 @@ public class DFSOutputStream extends FSO checksum.getBytesPerChecksum()); streamer = new DataStreamer(); } + this.fileEncryptionInfo = stat.getFileEncryptionInfo(); } static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src, @@ -2172,11 +2177,18 @@ public class DFSOutputStream extends FSO /** * Returns the size of a file as it was when this stream was opened */ - long getInitialLen() { + public long getInitialLen() { return initialFileSize; } /** + * @return the FileEncryptionInfo for this stream, or null if not encrypted. + */ + public FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + + /** * Returns the access token currently used by streamer, for testing only */ synchronized Token<BlockTokenIdentifier> getBlockToken() {
Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Aug 21 05:22:10 2014 @@ -71,6 +71,9 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.crypto.key.KeyProviderFactory; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -1722,4 +1725,39 @@ public class DFSUtil { } } } + + /** + * Creates a new KeyProviderCryptoExtension by wrapping the + * KeyProvider specified in the given Configuration. + * + * @param conf Configuration specifying a single, non-transient KeyProvider. + * @return new KeyProviderCryptoExtension, or null if no provider was found. + * @throws IOException if the KeyProvider is improperly specified in + * the Configuration + */ + public static KeyProviderCryptoExtension createKeyProviderCryptoExtension( + final Configuration conf) throws IOException { + final List<KeyProvider> providers = KeyProviderFactory.getProviders(conf); + if (providers == null || providers.size() == 0) { + return null; + } + if (providers.size() > 1) { + StringBuilder builder = new StringBuilder(); + builder.append("Found multiple KeyProviders but only one is permitted ["); + String prefix = " "; + for (KeyProvider kp: providers) { + builder.append(prefix + kp.toString()); + prefix = ", "; + } + builder.append("]"); + throw new IOException(builder.toString()); + } + KeyProviderCryptoExtension provider = KeyProviderCryptoExtension + .createKeyProviderCryptoExtension(providers.get(0)); + if (provider.isTransient()) { + throw new IOException("KeyProvider " + provider.toString() + + " was found but it is a transient provider."); + } + return provider; + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Thu Aug 21 05:22:10 2014 @@ -61,7 +61,6 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.client.HdfsAdmin; -import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; @@ -69,6 +68,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; @@ -291,8 +291,9 @@ public class DistributedFileSystem exten @Override public FSDataInputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataInputStream( - dfs.open(getPathName(p), bufferSize, verifyChecksum)); + final DFSInputStream dfsis = + dfs.open(getPathName(p), bufferSize, verifyChecksum); + return dfs.createWrappedInputStream(dfsis); } @Override public FSDataInputStream next(final FileSystem fs, final Path p) @@ -357,7 +358,7 @@ public class DistributedFileSystem exten : EnumSet.of(CreateFlag.CREATE), true, replication, blockSize, progress, bufferSize, null, favoredNodes); - return new HdfsDataOutputStream(out, statistics); + return dfs.createWrappedOutputStream(out, statistics); } @Override public HdfsDataOutputStream next(final FileSystem fs, final Path p) @@ -385,9 +386,10 @@ public class DistributedFileSystem exten @Override public FSDataOutputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataOutputStream(dfs.create(getPathName(p), permission, - cflags, replication, blockSize, progress, bufferSize, checksumOpt), - statistics); + final DFSOutputStream dfsos = dfs.create(getPathName(p), permission, + cflags, replication, blockSize, progress, bufferSize, + checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics); } @Override public FSDataOutputStream next(final FileSystem fs, final Path p) @@ -404,11 +406,12 @@ public class DistributedFileSystem exten short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { statistics.incrementWriteOps(1); - return new HdfsDataOutputStream(dfs.primitiveCreate( - getPathName(fixRelativePart(f)), - absolutePermission, flag, true, replication, blockSize, - progress, bufferSize, checksumOpt),statistics); - } + final DFSOutputStream dfsos = dfs.primitiveCreate( + getPathName(fixRelativePart(f)), + absolutePermission, flag, true, replication, blockSize, + progress, bufferSize, checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics); + } /** * Same as create(), except fails if parent directory doesn't already exist. @@ -428,9 +431,9 @@ public class DistributedFileSystem exten @Override public FSDataOutputStream doCall(final Path p) throws IOException, UnresolvedLinkException { - return new HdfsDataOutputStream(dfs.create(getPathName(p), permission, - flag, false, replication, blockSize, progress, bufferSize, null), - statistics); + final DFSOutputStream dfsos = dfs.create(getPathName(p), permission, + flag, false, replication, blockSize, progress, bufferSize, null); + return dfs.createWrappedOutputStream(dfsos, statistics); } @Override @@ -1822,6 +1825,25 @@ public class DistributedFileSystem exten }.resolve(this, absF); } + /* HDFS only */ + public void createEncryptionZone(Path path, String keyName) + throws IOException { + dfs.createEncryptionZone(getPathName(path), keyName); + } + + /* HDFS only */ + public EncryptionZone getEZForPath(Path path) + throws IOException { + Preconditions.checkNotNull(path); + return dfs.getEZForPath(getPathName(path)); + } + + /* HDFS only */ + public RemoteIterator<EncryptionZone> listEncryptionZones() + throws IOException { + return dfs.listEncryptionZones(); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag) throws IOException { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java Thu Aug 21 05:22:10 2014 @@ -49,9 +49,9 @@ public class XAttrHelper { Preconditions.checkNotNull(name, "XAttr name cannot be null."); final int prefixIndex = name.indexOf("."); - if (prefixIndex < 4) {// Prefix length is at least 4. + if (prefixIndex < 3) {// Prefix length is at least 3. throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } else if (prefixIndex == name.length() - 1) { throw new HadoopIllegalArgumentException("XAttr name cannot be empty."); } @@ -66,9 +66,11 @@ public class XAttrHelper { ns = NameSpace.SYSTEM; } else if (prefix.equals(NameSpace.SECURITY.toString().toLowerCase())) { ns = NameSpace.SECURITY; + } else if (prefix.equals(NameSpace.RAW.toString().toLowerCase())) { + ns = NameSpace.RAW; } else { throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name. substring(prefixIndex + 1)).setValue(value).build(); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java Thu Aug 21 05:22:10 2014 @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.client; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.EnumSet; @@ -33,7 +34,9 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.hdfs.tools.DFSAdmin; /** @@ -225,4 +228,51 @@ public class HdfsAdmin { public RemoteIterator<CachePoolEntry> listCachePools() throws IOException { return dfs.listCachePools(); } + + /** + * Create an encryption zone rooted at an empty existing directory, using the + * specified encryption key. An encryption zone has an associated encryption + * key used when reading and writing files within the zone. + * + * @param path The path of the root of the encryption zone. Must refer to + * an empty, existing directory. + * @param keyName Name of key available at the KeyProvider. + * @throws IOException if there was a general IO exception + * @throws AccessControlException if the caller does not have access to path + * @throws FileNotFoundException if the path does not exist + */ + public void createEncryptionZone(Path path, String keyName) + throws IOException, AccessControlException, FileNotFoundException { + dfs.createEncryptionZone(path, keyName); + } + + /** + * Get the path of the encryption zone for a given file or directory. + * + * @param path The path to get the ez for. + * + * @return The EncryptionZone of the ez, or null if path is not in an ez. + * @throws IOException if there was a general IO exception + * @throws AccessControlException if the caller does not have access to path + * @throws FileNotFoundException if the path does not exist + */ + public EncryptionZone getEncryptionZoneForPath(Path path) + throws IOException, AccessControlException, FileNotFoundException { + return dfs.getEZForPath(path); + } + + /** + * Returns a RemoteIterator which can be used to list the encryption zones + * in HDFS. For large numbers of encryption zones, the iterator will fetch + * the list of zones in a number of small batches. + * <p/> + * Since the list is fetched in batches, it does not represent a + * consistent snapshot of the entire list of encryption zones. + * <p/> + * This method can only be called by HDFS superusers. + */ + public RemoteIterator<EncryptionZone> listEncryptionZones() + throws IOException { + return dfs.listEncryptionZones(); + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java Thu Aug 21 05:22:10 2014 @@ -17,17 +17,21 @@ */ package org.apache.hadoop.hdfs.client; +import java.io.InputStream; import java.io.IOException; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import com.google.common.base.Preconditions; + /** * The Hdfs implementation of {@link FSDataInputStream}. */ @@ -38,25 +42,49 @@ public class HdfsDataInputStream extends super(in); } + public HdfsDataInputStream(CryptoInputStream in) throws IOException { + super(in); + Preconditions.checkArgument(in.getWrappedStream() instanceof DFSInputStream, + "CryptoInputStream should wrap a DFSInputStream"); + } + + private DFSInputStream getDFSInputStream() { + if (in instanceof CryptoInputStream) { + return (DFSInputStream) ((CryptoInputStream) in).getWrappedStream(); + } + return (DFSInputStream) in; + } + + /** + * Get a reference to the wrapped output stream. We always want to return the + * actual underlying InputStream, even when we're using a CryptoStream. e.g. + * in the delegated methods below. + * + * @return the underlying output stream + */ + public InputStream getWrappedStream() { + return in; + } + /** * Get the datanode from which the stream is currently reading. */ public DatanodeInfo getCurrentDatanode() { - return ((DFSInputStream) in).getCurrentDatanode(); + return getDFSInputStream().getCurrentDatanode(); } /** * Get the block containing the target position. */ public ExtendedBlock getCurrentBlock() { - return ((DFSInputStream) in).getCurrentBlock(); + return getDFSInputStream().getCurrentBlock(); } /** * Get the collection of blocks that has already been located. */ public synchronized List<LocatedBlock> getAllBlocks() throws IOException { - return ((DFSInputStream) in).getAllBlocks(); + return getDFSInputStream().getAllBlocks(); } /** @@ -66,7 +94,7 @@ public class HdfsDataInputStream extends * @return The visible length of the file. */ public long getVisibleLength() throws IOException { - return ((DFSInputStream) in).getFileLength(); + return getDFSInputStream().getFileLength(); } /** @@ -76,6 +104,6 @@ public class HdfsDataInputStream extends * bytes read through HdfsDataInputStream. */ public synchronized DFSInputStream.ReadStatistics getReadStatistics() { - return ((DFSInputStream) in).getReadStatistics(); + return getDFSInputStream().getReadStatistics(); } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java Thu Aug 21 05:22:10 2014 @@ -18,14 +18,18 @@ package org.apache.hadoop.hdfs.client; import java.io.IOException; +import java.io.OutputStream; import java.util.EnumSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSOutputStream; +import com.google.common.base.Preconditions; + /** * The Hdfs implementation of {@link FSDataOutputStream}. */ @@ -42,6 +46,18 @@ public class HdfsDataOutputStream extend this(out, stats, 0L); } + public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats, + long startPosition) throws IOException { + super(out, stats, startPosition); + Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream, + "CryptoOutputStream should wrap a DFSOutputStream"); + } + + public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats) + throws IOException { + this(out, stats, 0L); + } + /** * Get the actual number of replicas of the current block. * @@ -55,7 +71,11 @@ public class HdfsDataOutputStream extend * @return the number of valid replicas of the current block */ public synchronized int getCurrentBlockReplication() throws IOException { - return ((DFSOutputStream)getWrappedStream()).getCurrentBlockReplication(); + OutputStream wrappedStream = getWrappedStream(); + if (wrappedStream instanceof CryptoOutputStream) { + wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); + } + return ((DFSOutputStream) wrappedStream).getCurrentBlockReplication(); } /** @@ -67,14 +87,20 @@ public class HdfsDataOutputStream extend * @see FSDataOutputStream#hsync() */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { - ((DFSOutputStream) getWrappedStream()).hsync(syncFlags); + OutputStream wrappedStream = getWrappedStream(); + if (wrappedStream instanceof CryptoOutputStream) { + ((CryptoOutputStream) wrappedStream).flush(); + wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream(); + } + ((DFSOutputStream) wrappedStream).hsync(syncFlags); } public static enum SyncFlag { + /** - * When doing sync to DataNodes, also update the metadata (block - * length) in the NameNode + * When doing sync to DataNodes, also update the metadata (block length) in + * the NameNode. */ UPDATE_LENGTH; } -} \ No newline at end of file +} Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Thu Aug 21 05:22:10 2014 @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -188,7 +189,8 @@ public interface ClientProtocol { @AtMostOnce public HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable<CreateFlag> flag, - boolean createParent, short replication, long blockSize) + boolean createParent, short replication, long blockSize, + List<CipherSuite> cipherSuites) throws AccessControlException, AlreadyBeingCreatedException, DSQuotaExceededException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, @@ -1281,6 +1283,31 @@ public interface ClientProtocol { public AclStatus getAclStatus(String src) throws IOException; /** + * Create an encryption zone + */ + @AtMostOnce + public void createEncryptionZone(String src, String keyName) + throws IOException; + + /** + * Get the encryption zone for a path. + */ + @Idempotent + public EncryptionZoneWithId getEZForPath(String src) + throws IOException; + + /** + * Used to implement cursor-based batched listing of {@EncryptionZone}s. + * + * @param prevId ID of the last item in the previous batch. If there is no + * previous batch, a negative value can be used. + * @return Batch of encryption zones. + */ + @Idempotent + public BatchedEntries<EncryptionZoneWithId> listEncryptionZones( + long prevId) throws IOException; + + /** * Set xattr of a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". @@ -1321,7 +1348,6 @@ public interface ClientProtocol { * Refer to the HDFS extended attributes user documentation for details. * * @param src file or directory - * @param xAttrs xAttrs to get * @return List<XAttr> <code>XAttr</code> list * @throws IOException */ Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java Thu Aug 21 05:22:10 2014 @@ -21,6 +21,7 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -45,6 +46,8 @@ public class HdfsFileStatus { private final String group; private final long fileId; + private final FileEncryptionInfo feInfo; + // Used by dir, not including dot and dotdot. Always zero for a regular file. private final int childrenNum; private final byte storagePolicy; @@ -64,11 +67,13 @@ public class HdfsFileStatus { * @param group the group of the path * @param path the local name in java UTF8 encoding the same as that in-memory * @param fileId the file id + * @param feInfo the file's encryption info */ public HdfsFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, - byte[] path, long fileId, int childrenNum, byte storagePolicy) { + byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, + byte storagePolicy) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -86,6 +91,7 @@ public class HdfsFileStatus { this.path = path; this.fileId = fileId; this.childrenNum = childrenNum; + this.feInfo = feInfo; this.storagePolicy = storagePolicy; } @@ -240,6 +246,10 @@ public class HdfsFileStatus { return fileId; } + public final FileEncryptionInfo getFileEncryptionInfo() { + return feInfo; + } + public final int getChildrenNum() { return childrenNum; } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java Thu Aug 21 05:22:10 2014 @@ -21,6 +21,7 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -51,15 +52,16 @@ public class HdfsLocatedFileStatus exten * @param path local path name in java UTF8 format * @param fileId the file id * @param locations block locations + * @param feInfo file encryption info */ public HdfsLocatedFileStatus(long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, LocatedBlocks locations, - int childrenNum, byte storagePolicy) { + int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) { super(length, isdir, block_replication, blocksize, modification_time, access_time, permission, owner, group, symlink, path, fileId, - childrenNum, storagePolicy); + childrenNum, feInfo, storagePolicy); this.locations = locations; } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java Thu Aug 21 05:22:10 2014 @@ -23,6 +23,7 @@ import java.util.Comparator; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileEncryptionInfo; /** * Collection of blocks with their locations and the file length. @@ -35,22 +36,23 @@ public class LocatedBlocks { private final boolean underConstruction; private LocatedBlock lastLocatedBlock = null; private boolean isLastBlockComplete = false; + private FileEncryptionInfo fileEncryptionInfo = null; public LocatedBlocks() { fileLength = 0; blocks = null; underConstruction = false; } - - /** public Constructor */ + public LocatedBlocks(long flength, boolean isUnderConstuction, - List<LocatedBlock> blks, - LocatedBlock lastBlock, boolean isLastBlockCompleted) { + List<LocatedBlock> blks, LocatedBlock lastBlock, + boolean isLastBlockCompleted, FileEncryptionInfo feInfo) { fileLength = flength; blocks = blks; underConstruction = isUnderConstuction; this.lastLocatedBlock = lastBlock; this.isLastBlockComplete = isLastBlockCompleted; + this.fileEncryptionInfo = feInfo; } /** @@ -92,13 +94,20 @@ public class LocatedBlocks { } /** - * Return ture if file was under construction when - * this LocatedBlocks was constructed, false otherwise. + * Return true if file was under construction when this LocatedBlocks was + * constructed, false otherwise. */ public boolean isUnderConstruction() { return underConstruction; } - + + /** + * @return the FileEncryptionInfo for the LocatedBlocks + */ + public FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + /** * Find block containing specified offset. * Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java Thu Aug 21 05:22:10 2014 @@ -62,7 +62,7 @@ public class SnapshottableDirectoryStatu int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, access_time, permission, owner, group, null, localName, inodeId, - childrenNum, BlockStoragePolicy.ID_UNSPECIFIED); + childrenNum, null, BlockStoragePolicy.ID_UNSPECIFIED); this.snapshotNumber = snapshotNumber; this.snapshotQuota = snapshotQuota; this.parentFullPath = parentFullPath; Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java Thu Aug 21 05:22:10 2014 @@ -162,8 +162,10 @@ public final class DataTransferSaslUtil Configuration saslPropsResolverConf = new Configuration(conf); saslPropsResolverConf.set(HADOOP_RPC_PROTECTION, qops); Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass( - DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, + HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, SaslPropertiesResolver.class, SaslPropertiesResolver.class); + resolverClass = conf.getClass(DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY, + resolverClass, SaslPropertiesResolver.class); saslPropsResolverConf.setClass(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, resolverClass, SaslPropertiesResolver.class); SaslPropertiesResolver resolver = SaslPropertiesResolver.getInstance( Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Thu Aug 21 05:22:10 2014 @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -178,6 +179,12 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; @@ -380,7 +387,8 @@ public class ClientNamenodeProtocolServe HdfsFileStatus result = server.create(req.getSrc(), PBHelper.convert(req.getMasked()), req.getClientName(), PBHelper.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(), - (short) req.getReplication(), req.getBlockSize()); + (short) req.getReplication(), req.getBlockSize(), + PBHelper.convertCipherSuiteProtos(req.getCipherSuitesList())); if (result != null) { return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result)) @@ -1305,6 +1313,52 @@ public class ClientNamenodeProtocolServe } @Override + public CreateEncryptionZoneResponseProto createEncryptionZone( + RpcController controller, CreateEncryptionZoneRequestProto req) + throws ServiceException { + try { + server.createEncryptionZone(req.getSrc(), req.getKeyName()); + return CreateEncryptionZoneResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetEZForPathResponseProto getEZForPath( + RpcController controller, GetEZForPathRequestProto req) + throws ServiceException { + try { + GetEZForPathResponseProto.Builder builder = + GetEZForPathResponseProto.newBuilder(); + final EncryptionZoneWithId ret = server.getEZForPath(req.getSrc()); + builder.setZone(PBHelper.convert(ret)); + return builder.build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ListEncryptionZonesResponseProto listEncryptionZones( + RpcController controller, ListEncryptionZonesRequestProto req) + throws ServiceException { + try { + BatchedEntries<EncryptionZoneWithId> entries = server + .listEncryptionZones(req.getId()); + ListEncryptionZonesResponseProto.Builder builder = + ListEncryptionZonesResponseProto.newBuilder(); + builder.setHasMore(entries.hasMore()); + for (int i=0; i<entries.size(); i++) { + builder.addZones(PBHelper.convert(entries.get(i))); + } + return builder.build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override public SetXAttrResponseProto setXAttr(RpcController controller, SetXAttrRequestProto req) throws ServiceException { try { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Thu Aug 21 05:22:10 2014 @@ -24,8 +24,10 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -52,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; @@ -149,6 +152,10 @@ import org.apache.hadoop.hdfs.protocol.p import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto; @@ -175,6 +182,11 @@ import org.apache.hadoop.security.token. import com.google.protobuf.ByteString; import com.google.protobuf.ServiceException; + +import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; +import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos + .EncryptionZoneWithIdProto; + /** * This class forwards NN's ClientProtocol calls as RPC calls to the NN server * while translating from the parameter types used in ClientProtocol to the @@ -252,21 +264,25 @@ public class ClientNamenodeProtocolTrans @Override public HdfsFileStatus create(String src, FsPermission masked, String clientName, EnumSetWritable<CreateFlag> flag, - boolean createParent, short replication, long blockSize) + boolean createParent, short replication, long blockSize, + List<CipherSuite> cipherSuites) throws AccessControlException, AlreadyBeingCreatedException, DSQuotaExceededException, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, ParentNotDirectoryException, SafeModeException, UnresolvedLinkException, IOException { - CreateRequestProto req = CreateRequestProto.newBuilder() + CreateRequestProto.Builder builder = CreateRequestProto.newBuilder() .setSrc(src) .setMasked(PBHelper.convert(masked)) .setClientName(clientName) .setCreateFlag(PBHelper.convertCreateFlag(flag)) .setCreateParent(createParent) .setReplication(replication) - .setBlockSize(blockSize) - .build(); + .setBlockSize(blockSize); + if (cipherSuites != null) { + builder.addAllCipherSuites(PBHelper.convertCipherSuites(cipherSuites)); + } + CreateRequestProto req = builder.build(); try { CreateResponseProto res = rpcProxy.create(null, req); return res.hasFs() ? PBHelper.convert(res.getFs()) : null; @@ -1295,7 +1311,62 @@ public class ClientNamenodeProtocolTrans throw ProtobufHelper.getRemoteException(e); } } - + + @Override + public void createEncryptionZone(String src, String keyName) + throws IOException { + final CreateEncryptionZoneRequestProto.Builder builder = + CreateEncryptionZoneRequestProto.newBuilder(); + builder.setSrc(src); + if (keyName != null && !keyName.isEmpty()) { + builder.setKeyName(keyName); + } + CreateEncryptionZoneRequestProto req = builder.build(); + try { + rpcProxy.createEncryptionZone(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public EncryptionZoneWithId getEZForPath(String src) + throws IOException { + final GetEZForPathRequestProto.Builder builder = + GetEZForPathRequestProto.newBuilder(); + builder.setSrc(src); + final GetEZForPathRequestProto req = builder.build(); + try { + final EncryptionZonesProtos.GetEZForPathResponseProto response = + rpcProxy.getEZForPath(null, req); + return PBHelper.convert(response.getZone()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public BatchedEntries<EncryptionZoneWithId> listEncryptionZones(long id) + throws IOException { + final ListEncryptionZonesRequestProto req = + ListEncryptionZonesRequestProto.newBuilder() + .setId(id) + .build(); + try { + EncryptionZonesProtos.ListEncryptionZonesResponseProto response = + rpcProxy.listEncryptionZones(null, req); + List<EncryptionZoneWithId> elements = + Lists.newArrayListWithCapacity(response.getZonesCount()); + for (EncryptionZoneWithIdProto p : response.getZonesList()) { + elements.add(PBHelper.convert(p)); + } + return new BatchedListEntries<EncryptionZoneWithId>(elements, + response.getHasMore()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag) throws IOException { Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Thu Aug 21 05:22:10 2014 @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.protocolPB; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos + .EncryptionZoneWithIdProto; import java.io.EOFException; import java.io.IOException; @@ -52,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CachePoolStats; +import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -59,7 +62,9 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdfs.protocol.FsAclPermission; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; @@ -1179,7 +1184,9 @@ public class PBHelper { lb.getFileLength(), lb.getUnderConstruction(), PBHelper.convertLocatedBlock(lb.getBlocksList()), lb.hasLastBlock() ? PBHelper.convert(lb.getLastBlock()) : null, - lb.getIsLastBlockComplete()); + lb.getIsLastBlockComplete(), + lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) : + null); } public static LocatedBlocksProto convert(LocatedBlocks lb) { @@ -1191,6 +1198,9 @@ public class PBHelper { if (lb.getLastLocatedBlock() != null) { builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock())); } + if (lb.getFileEncryptionInfo() != null) { + builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo())); + } return builder.setFileLength(lb.getFileLength()) .setUnderConstruction(lb.isUnderConstruction()) .addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks())) @@ -1317,6 +1327,7 @@ public class PBHelper { fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID, fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null, fs.hasChildrenNum() ? fs.getChildrenNum() : -1, + fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() : BlockStoragePolicy.ID_UNSPECIFIED); } @@ -1369,6 +1380,9 @@ public class PBHelper { if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); } + if (fs.getFileEncryptionInfo() != null) { + builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo())); + } if (fs instanceof HdfsLocatedFileStatus) { final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs; LocatedBlocks locations = lfs.getBlockLocations(); @@ -2262,7 +2276,7 @@ public class PBHelper { } return xAttrs; } - + public static List<XAttr> convert(GetXAttrsResponseProto a) { List<XAttrProto> xAttrs = a.getXAttrsList(); return convertXAttrs(xAttrs); @@ -2293,6 +2307,18 @@ public class PBHelper { return builder.build(); } + public static EncryptionZoneWithIdProto convert(EncryptionZoneWithId zone) { + return EncryptionZoneWithIdProto.newBuilder() + .setId(zone.getId()) + .setKeyName(zone.getKeyName()) + .setPath(zone.getPath()).build(); + } + + public static EncryptionZoneWithId convert(EncryptionZoneWithIdProto proto) { + return new EncryptionZoneWithId(proto.getPath(), proto.getKeyName(), + proto.getId()); + } + public static ShortCircuitShmSlotProto convert(SlotId slotId) { return ShortCircuitShmSlotProto.newBuilder(). setShmId(convert(slotId.getShmId())). @@ -2316,5 +2342,75 @@ public class PBHelper { public static ShmId convert(ShortCircuitShmIdProto shmId) { return new ShmId(shmId.getHi(), shmId.getLo()); } -} + public static HdfsProtos.CipherSuite convert(CipherSuite suite) { + switch (suite) { + case UNKNOWN: + return HdfsProtos.CipherSuite.UNKNOWN; + case AES_CTR_NOPADDING: + return HdfsProtos.CipherSuite.AES_CTR_NOPADDING; + default: + return null; + } + } + + public static CipherSuite convert(HdfsProtos.CipherSuite proto) { + switch (proto) { + case AES_CTR_NOPADDING: + return CipherSuite.AES_CTR_NOPADDING; + default: + // Set to UNKNOWN and stash the unknown enum value + CipherSuite suite = CipherSuite.UNKNOWN; + suite.setUnknownValue(proto.getNumber()); + return suite; + } + } + + public static List<HdfsProtos.CipherSuite> convertCipherSuites + (List<CipherSuite> suites) { + if (suites == null) { + return null; + } + List<HdfsProtos.CipherSuite> protos = + Lists.newArrayListWithCapacity(suites.size()); + for (CipherSuite suite : suites) { + protos.add(convert(suite)); + } + return protos; + } + + public static List<CipherSuite> convertCipherSuiteProtos( + List<HdfsProtos.CipherSuite> protos) { + List<CipherSuite> suites = Lists.newArrayListWithCapacity(protos.size()); + for (HdfsProtos.CipherSuite proto : protos) { + suites.add(convert(proto)); + } + return suites; + } + + public static HdfsProtos.FileEncryptionInfoProto convert( + FileEncryptionInfo info) { + if (info == null) { + return null; + } + return HdfsProtos.FileEncryptionInfoProto.newBuilder() + .setSuite(convert(info.getCipherSuite())) + .setKey(getByteString(info.getEncryptedDataEncryptionKey())) + .setIv(getByteString(info.getIV())) + .setEzKeyVersionName(info.getEzKeyVersionName()) + .build(); + } + + public static FileEncryptionInfo convert( + HdfsProtos.FileEncryptionInfoProto proto) { + if (proto == null) { + return null; + } + CipherSuite suite = convert(proto.getSuite()); + byte[] key = proto.getKey().toByteArray(); + byte[] iv = proto.getIv().toByteArray(); + String ezKeyVersionName = proto.getEzKeyVersionName(); + return new FileEncryptionInfo(suite, key, iv, ezKeyVersionName); + } + +} Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Thu Aug 21 05:22:10 2014 @@ -53,6 +53,8 @@ import org.apache.hadoop.hdfs.protocol.B import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.fs.FileEncryptionInfo; + import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; @@ -846,14 +848,15 @@ public class BlockManager { public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks, final long fileSizeExcludeBlocksUnderConstruction, final boolean isFileUnderConstruction, final long offset, - final long length, final boolean needBlockToken, final boolean inSnapshot) + final long length, final boolean needBlockToken, + final boolean inSnapshot, FileEncryptionInfo feInfo) throws IOException { assert namesystem.hasReadLock(); if (blocks == null) { return null; } else if (blocks.length == 0) { return new LocatedBlocks(0, isFileUnderConstruction, - Collections.<LocatedBlock>emptyList(), null, false); + Collections.<LocatedBlock>emptyList(), null, false, feInfo); } else { if (LOG.isDebugEnabled()) { LOG.debug("blocks = " + java.util.Arrays.asList(blocks)); @@ -878,7 +881,7 @@ public class BlockManager { } return new LocatedBlocks( fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction, - locatedblocks, lastlb, isComplete); + locatedblocks, lastlb, isComplete, feInfo); } } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java Thu Aug 21 05:22:10 2014 @@ -294,5 +294,10 @@ public final class HdfsServerConstants { public static final String NAMENODE_LEASE_HOLDER = "HDFS_NameNode"; public static final long NAMENODE_LEASE_RECHECK_INTERVAL = 2000; + + public static final String CRYPTO_XATTR_ENCRYPTION_ZONE = + "raw.hdfs.crypto.encryption.zone"; + public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO = + "raw.hdfs.crypto.file.encryption.info"; } Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Thu Aug 21 05:22:10 2014 @@ -576,7 +576,9 @@ class DataXceiver extends Receiver imple // forward the original version of the block to downstream mirrors, so // make a copy here. final ExtendedBlock originalBlock = new ExtendedBlock(block); - block.setNumBytes(dataXceiverServer.estimateBlockSize); + if (block.getNumBytes() == 0) { + block.setNumBytes(dataXceiverServer.estimateBlockSize); + } LOG.info("Receiving " + block + " src: " + remoteAddress + " dest: " + localAddress); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java Thu Aug 21 05:22:10 2014 @@ -100,11 +100,8 @@ class DataXceiverServer implements Runna /** * We need an estimate for block size to check if the disk partition has - * enough space. For now we set it to be the default block size set - * in the server side configuration, which is not ideal because the - * default block size should be a client-size configuration. - * A better solution is to include in the header the estimated block size, - * i.e. either the actual block size or the default block size. + * enough space. Newer clients pass the expected block size to the DataNode. + * For older clients we just use the server-side default block size. */ final long estimateBlockSize;
