[31/37] hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah
HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46 Branch: refs/heads/HDFS-7240 Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73 Parents: 7fd287b Author: Kihwal LeeAuthored: Mon Jan 29 17:22:29 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:23:29 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 101 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 85 ++--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 403 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 92bb99e..2497c40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -38,7 +38,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK")) { +cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis, +getKeyProvider(), feInfo, getConfiguration()); + } return new
[15/50] [abbrv] hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah
HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46 Branch: refs/heads/YARN-6592 Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73 Parents: 7fd287b Author: Kihwal LeeAuthored: Mon Jan 29 17:22:29 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:23:29 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 101 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 85 ++--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 403 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 92bb99e..2497c40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -38,7 +38,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK")) { +cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis, +getKeyProvider(), feInfo, getConfiguration()); + } return new
hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 802057df3 -> d9132bf5e HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9132bf5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9132bf5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9132bf5 Branch: refs/heads/branch-2.8 Commit: d9132bf5ee8a2054d2d9f3e9b3e26cce23d6bcea Parents: 802057d Author: Kihwal LeeAuthored: Mon Jan 29 18:11:19 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 18:11:19 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 100 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 79 +--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 398 insertions(+), 88 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9132bf5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 56335a4..de00da9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -41,7 +41,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CacheFlag; @@ -980,45 +977,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK")) { +cryptoIn =
hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2.9 d70ca9959 -> a15df67f6 HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah. (cherry picked from commit eda786ea12db6b3ca8d6b0565c4ebdeab28b3cf6) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a15df67f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a15df67f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a15df67f Branch: refs/heads/branch-2.9 Commit: a15df67f6508eab3dbaa149bebd040fa506cf1df Parents: d70ca99 Author: Kihwal LeeAuthored: Mon Jan 29 17:57:54 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:57:54 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 99 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 83 +--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 399 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a15df67f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index be4de50..fffaafb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -41,7 +41,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CacheFlag; @@ -980,45 +977,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), -
hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.
Repository: hadoop Updated Branches: refs/heads/branch-2 987a8972a -> eda786ea1 HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eda786ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eda786ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eda786ea Branch: refs/heads/branch-2 Commit: eda786ea12db6b3ca8d6b0565c4ebdeab28b3cf6 Parents: 987a897 Author: Kihwal LeeAuthored: Mon Jan 29 17:52:04 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:52:04 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 99 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 83 +--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 399 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eda786ea/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 3e92340..ad6aaad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -41,7 +41,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CacheFlag; @@ -981,45 +978,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK")) { +cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis, +
hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah
Repository: hadoop Updated Branches: refs/heads/branch-3.0 95a96b13e -> 673200ac1 HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah (cherry picked from commit fde95d463c3123b315b3d07cb5b7b7dc19f7cb73) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/673200ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/673200ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/673200ac Branch: refs/heads/branch-3.0 Commit: 673200ac1e1cc6a8cf4f847db21df9205131b6b6 Parents: 95a96b1 Author: Kihwal LeeAuthored: Mon Jan 29 17:25:30 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:25:30 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 101 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 85 ++--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 403 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/673200ac/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index e08d403..fa2a2bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -38,7 +38,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -909,45 +906,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK"))
hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah
Repository: hadoop Updated Branches: refs/heads/trunk 7fd287b4a -> fde95d463 HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46 Branch: refs/heads/trunk Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73 Parents: 7fd287b Author: Kihwal LeeAuthored: Mon Jan 29 17:22:29 2018 -0600 Committer: Kihwal Lee Committed: Mon Jan 29 17:23:29 2018 -0600 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 48 ++--- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 41 .../hadoop/hdfs/web/WebHdfsFileSystem.java | 101 -- .../hdfs/web/TestWebHdfsContentLength.java | 2 + .../web/resources/NamenodeWebHdfsMethods.java | 85 ++--- .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++ .../web/resources/TestWebHdfsDataLocality.java | 23 ++- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 1 - .../hadoop/hdfs/web/TestWebHdfsTokens.java | 4 +- 9 files changed, 403 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 92bb99e..2497c40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -38,7 +38,6 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** - * Decrypts a EDEK by consulting the KeyProvider. - */ - private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo - feInfo) throws IOException { -try (TraceScope ignored = tracer.newScope("decryptEDEK")) { - KeyProvider provider = getKeyProvider(); - if (provider == null) { -throw new IOException("No KeyProvider is configured, cannot access" + -" an encrypted file"); - } - EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( - feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(), - feInfo.getEncryptedDataEncryptionKey()); - try { -KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension -.createKeyProviderCryptoExtension(provider); -return cryptoProvider.decryptEncryptedKey(ekv); - } catch (GeneralSecurityException e) { -throw new IOException(e); - } -} - } - - /** * Wraps the stream in a CryptoInputStream if the underlying file is * encrypted. */ public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) throws IOException { -final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); +FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); if (feInfo != null) { - // File is encrypted, wrap the stream in a crypto stream. - // Currently only one version, so no special logic based on the version # - HdfsKMSUtil.getCryptoProtocolVersion(feInfo); - final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo); - final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); - final CryptoInputStream cryptoIn = - new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), - feInfo.getIV()); + CryptoInputStream cryptoIn; + try (TraceScope ignored = getTracer().newScope("decryptEDEK")) { +cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis, +