HDFS-10918. Add a tool to get FileEncryptionInfo from CLI. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/853d65a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/853d65a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/853d65a1 Branch: refs/heads/HADOOP-12756 Commit: 853d65a157362661ccab10379c2d82e780382f83 Parents: 736d33c Author: Xiao Chen <x...@apache.org> Authored: Mon Oct 3 16:01:54 2016 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Mon Oct 3 16:01:54 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/fs/FileEncryptionInfo.java | 21 +++++ .../hadoop/hdfs/DistributedFileSystem.java | 30 +++++++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 14 +++ .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 51 ++++++++++- .../src/site/markdown/TransparentEncryption.md | 16 ++++ .../src/test/resources/testCryptoConf.xml | 90 ++++++++++++++++++++ 6 files changed, 221 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java index 00ddfe8..1129e07 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java @@ -121,4 +121,25 @@ public class FileEncryptionInfo { builder.append("}"); return builder.toString(); } + + /** + * A frozen version of {@link #toString()} to be backward compatible. + * When backward compatibility is not needed, use {@link #toString()}, which + * provides more info and is supposed to evolve. + * Don't change this method except for major revisions. + * + * NOTE: + * Currently this method is used by CLI for backward compatibility. + */ + public String toStringStable() { + StringBuilder builder = new StringBuilder("{"); + builder.append("cipherSuite: " + cipherSuite); + builder.append(", cryptoProtocolVersion: " + version); + builder.append(", edek: " + Hex.encodeHexString(edek)); + builder.append(", iv: " + Hex.encodeHexString(iv)); + builder.append(", keyName: " + keyName); + builder.append(", ezKeyVersionName: " + ezKeyVersionName); + builder.append("}"); + return builder.toString(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 24ffb40..548815f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSLinkResolver; import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemLinkResolver; @@ -2204,6 +2205,35 @@ public class DistributedFileSystem extends FileSystem { return dfs.listEncryptionZones(); } + /* HDFS only */ + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + Path absF = fixRelativePart(path); + return new FileSystemLinkResolver<FileEncryptionInfo>() { + @Override + public FileEncryptionInfo doCall(final Path p) throws IOException { + final HdfsFileStatus fi = dfs.getFileInfo(getPathName(p)); + if (fi == null) { + throw new FileNotFoundException("File does not exist: " + p); + } + return fi.getFileEncryptionInfo(); + } + + @Override + public FileEncryptionInfo next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + return myDfs.getFileEncryptionInfo(p); + } + throw new UnsupportedOperationException( + "Cannot call getFileEncryptionInfo" + + " on a symlink to a non-DistributedFileSystem: " + path + + " -> " + p); + } + }.resolve(this, absF); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index bac2809..b12fe01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -354,6 +355,19 @@ public class HdfsAdmin { } /** + * Returns the FileEncryptionInfo on the HdfsFileStatus for the given path. + * The return value can be null if the path points to a directory, or a file + * that is not in an encryption zone. + * + * @throws FileNotFoundException if the path does not exist. + * @throws AccessControlException if no execute permission on parent path. + */ + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + return dfs.getFileEncryptionInfo(path); + } + + /** * Exposes a stream of namesystem events. Only events occurring after the * stream is created are available. * See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream} http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java index 06389a1..b78da31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -193,6 +194,53 @@ public class CryptoAdmin extends Configured implements Tool { } } + private static class GetFileEncryptionInfoCommand + implements AdminHelper.Command { + @Override + public String getName() { + return "-getFileEncryptionInfo"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + " -path <path>]\n"; + } + + @Override + public String getLongUsage() { + final TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("<path>", "The path to the file to show encryption info."); + return getShortUsage() + "\n" + "Get encryption info of a file.\n\n" + + listing.toString(); + } + + @Override + public int run(Configuration conf, List<String> args) throws IOException { + final String path = StringUtils.popOptionWithArgument("-path", args); + + if (!args.isEmpty()) { + System.err.println("Can't understand argument: " + args.get(0)); + return 1; + } + + final HdfsAdmin admin = + new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); + try { + final FileEncryptionInfo fei = + admin.getFileEncryptionInfo(new Path(path)); + if (fei == null) { + System.out.println("No FileEncryptionInfo found for path " + path); + return 2; + } + System.out.println(fei.toStringStable()); + } catch (IOException e) { + System.err.println(prettifyException(e)); + return 3; + } + return 0; + } + } + private static class ProvisionTrashCommand implements AdminHelper.Command { @Override public String getName() { @@ -237,6 +285,7 @@ public class CryptoAdmin extends Configured implements Tool { private static final AdminHelper.Command[] COMMANDS = { new CreateZoneCommand(), new ListZonesCommand(), - new ProvisionTrashCommand() + new ProvisionTrashCommand(), + new GetFileEncryptionInfoCommand() }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index ee98df8..e7d9f1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -29,6 +29,8 @@ Transparent Encryption in HDFS * [crypto command-line interface](#crypto_command-line_interface) * [createZone](#createZone) * [listZones](#listZones) + * [provisionTrash](#provisionTrash) + * [getFileEncryptionInfo](#getFileEncryptionInfo) * [Example usage](#Example_usage) * [Distcp considerations](#Distcp_considerations) * [Running as the superuser](#Running_as_the_superuser) @@ -189,6 +191,16 @@ Provision a trash directory for an encryption zone. |:---- |:---- | | *path* | The path to the root of the encryption zone. | +### <a name="getFileEncryptionInfo"></a>getFileEncryptionInfo + +Usage: `[-getFileEncryptionInfo -path <path>]` + +Get encryption information from a file. This can be used to find out whether a file is being encrypted, and the key name / key version used to encrypt it. + +| | | +|:---- |:---- | +| *path* | The path of the file to get encryption information. | + <a name="Example_usage"></a>Example usage ------------- @@ -208,6 +220,10 @@ These instructions assume that you are running as the normal user or HDFS superu hadoop fs -put helloWorld /zone hadoop fs -cat /zone/helloWorld + # As the normal user, get encryption information from the file + hdfs crypto -getFileEncryptionInfo -path /zone/helloWorld + # console output: {cipherSuite: {name: AES/CTR/NoPadding, algorithmBlockSize: 16}, cryptoProtocolVersion: CryptoProtocolVersion{description='Encryption zones', version=1, unknownValue=null}, edek: 2010d301afbd43b58f10737ce4e93b39, iv: ade2293db2bab1a2e337f91361304cb3, keyName: mykey, ezKeyVersionName: mykey@0} + <a name="Distcp_considerations"></a>Distcp considerations --------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/853d65a1/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml index ddd4adc..0294368 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml @@ -388,5 +388,95 @@ </comparator> </comparators> </test> + + <test> + <description>Test success of getFileEncryptionInfo on a EZ file</description> + <test-commands> + <command>-fs NAMENODE -mkdir /src</command> + <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command> + <command>-fs NAMENODE -touchz /src/file</command> + <crypto-admin-command>-getFileEncryptionInfo -path /src/file</crypto-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /src</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>keyName: myKey, ezKeyVersionName: myKey@0</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>Test failure of getFileEncryptionInfo on a non-EZ file</description> + <test-commands> + <command>-fs NAMENODE -mkdir /src</command> + <command>-fs NAMENODE -touchz /src/cleartext</command> + <crypto-admin-command>-getFileEncryptionInfo -path /src/cleartext</crypto-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /src</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>No FileEncryptionInfo found for path</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>Test failure of getFileEncryptionInfo on a non-exist file</description> + <test-commands> + <crypto-admin-command>-getFileEncryptionInfo -path /src/file</crypto-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /src</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>FileNotFoundException:</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>Test failure of getFileEncryptionInfo on a EZ dir</description> + <test-commands> + <command>-fs NAMENODE -mkdir /src</command> + <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command> + <crypto-admin-command>-getFileEncryptionInfo -path /src</crypto-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /src</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>No FileEncryptionInfo found for path </expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>Test failure of getFileEncryptionInfo on a EZ subdir</description> + <test-commands> + <command>-fs NAMENODE -mkdir /src</command> + <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command> + <command>-fs NAMENODE -mkdir /src/dir</command> + <crypto-admin-command>-getFileEncryptionInfo -path /src/dir</crypto-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /src</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>No FileEncryptionInfo found for path </expected-output> + </comparator> + </comparators> + </test> </tests> </configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
