[31/37] hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah

2018-01-30 Thread aengineer 
HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46

Branch: refs/heads/HDFS-7240
Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73
Parents: 7fd287b
Author: Kihwal Lee 
Authored: Mon Jan 29 17:22:29 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:23:29 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  | 101 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  85 ++---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 403 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 92bb99e..2497c40 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -38,7 +38,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
@@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK")) {
+cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis,
+getKeyProvider(), feInfo, getConfiguration());
+  }
   return new

[15/50] [abbrv] hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah

2018-01-30 Thread asuresh 
HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46

Branch: refs/heads/YARN-6592
Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73
Parents: 7fd287b
Author: Kihwal Lee 
Authored: Mon Jan 29 17:22:29 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:23:29 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  | 101 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  85 ++---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 403 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 92bb99e..2497c40 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -38,7 +38,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
@@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK")) {
+cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis,
+getKeyProvider(), feInfo, getConfiguration());
+  }
   return new

hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.

2018-01-29 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 802057df3 -> d9132bf5e


HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9132bf5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9132bf5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9132bf5

Branch: refs/heads/branch-2.8
Commit: d9132bf5ee8a2054d2d9f3e9b3e26cce23d6bcea
Parents: 802057d
Author: Kihwal Lee 
Authored: Mon Jan 29 18:11:19 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 18:11:19 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  | 100 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  79 +---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 398 insertions(+), 88 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9132bf5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 56335a4..de00da9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -41,7 +41,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.BlockStorageLocation;
 import org.apache.hadoop.fs.CacheFlag;
@@ -980,45 +977,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK")) {
+cryptoIn =

hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.

2018-01-29 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 d70ca9959 -> a15df67f6


HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah.

(cherry picked from commit eda786ea12db6b3ca8d6b0565c4ebdeab28b3cf6)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a15df67f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a15df67f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a15df67f

Branch: refs/heads/branch-2.9
Commit: a15df67f6508eab3dbaa149bebd040fa506cf1df
Parents: d70ca99
Author: Kihwal Lee 
Authored: Mon Jan 29 17:57:54 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:57:54 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  |  99 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  83 +---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 399 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a15df67f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index be4de50..fffaafb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -41,7 +41,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.BlockStorageLocation;
 import org.apache.hadoop.fs.CacheFlag;
@@ -980,45 +977,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-

hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah.

2018-01-29 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 987a8972a -> eda786ea1


HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eda786ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eda786ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eda786ea

Branch: refs/heads/branch-2
Commit: eda786ea12db6b3ca8d6b0565c4ebdeab28b3cf6
Parents: 987a897
Author: Kihwal Lee 
Authored: Mon Jan 29 17:52:04 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:52:04 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  |  99 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  83 +---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 399 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eda786ea/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3e92340..ad6aaad 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -41,7 +41,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -66,8 +65,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.BlockStorageLocation;
 import org.apache.hadoop.fs.CacheFlag;
@@ -981,45 +978,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK")) {
+cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis,
+

hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah

2018-01-29 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 95a96b13e -> 673200ac1


HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah

(cherry picked from commit fde95d463c3123b315b3d07cb5b7b7dc19f7cb73)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/673200ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/673200ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/673200ac

Branch: refs/heads/branch-3.0
Commit: 673200ac1e1cc6a8cf4f847db21df9205131b6b6
Parents: 95a96b1
Author: Kihwal Lee 
Authored: Mon Jan 29 17:25:30 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:25:30 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  | 101 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  85 ++---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 403 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/673200ac/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index e08d403..fa2a2bd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -38,7 +38,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
@@ -909,45 +906,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK"))

hadoop git commit: HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed by Rushabh S Shah

2018-01-29 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 7fd287b4a -> fde95d463


HDFS-12574. Add CryptoInputStream to WebHdfsFileSystem read call. Contributed 
by Rushabh S Shah


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde95d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde95d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde95d46

Branch: refs/heads/trunk
Commit: fde95d463c3123b315b3d07cb5b7b7dc19f7cb73
Parents: 7fd287b
Author: Kihwal Lee 
Authored: Mon Jan 29 17:22:29 2018 -0600
Committer: Kihwal Lee 
Committed: Mon Jan 29 17:23:29 2018 -0600

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  48 ++---
 .../org/apache/hadoop/hdfs/HdfsKMSUtil.java |  41 
 .../hadoop/hdfs/web/WebHdfsFileSystem.java  | 101 --
 .../hdfs/web/TestWebHdfsContentLength.java  |   2 +
 .../web/resources/NamenodeWebHdfsMethods.java   |  85 ++---
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 188 +++
 .../web/resources/TestWebHdfsDataLocality.java  |  23 ++-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   1 -
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   4 +-
 9 files changed, 403 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde95d46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 92bb99e..2497c40 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -38,7 +38,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -62,8 +61,6 @@ import org.apache.hadoop.crypto.CryptoInputStream;
 import org.apache.hadoop.crypto.CryptoOutputStream;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import 
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
@@ -911,45 +908,18 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   }
 
   /**
-   * Decrypts a EDEK by consulting the KeyProvider.
-   */
-  private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
-  feInfo) throws IOException {
-try (TraceScope ignored = tracer.newScope("decryptEDEK")) {
-  KeyProvider provider = getKeyProvider();
-  if (provider == null) {
-throw new IOException("No KeyProvider is configured, cannot access" +
-" an encrypted file");
-  }
-  EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-  feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-  feInfo.getEncryptedDataEncryptionKey());
-  try {
-KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-.createKeyProviderCryptoExtension(provider);
-return cryptoProvider.decryptEncryptedKey(ekv);
-  } catch (GeneralSecurityException e) {
-throw new IOException(e);
-  }
-}
-  }
-
-  /**
* Wraps the stream in a CryptoInputStream if the underlying file is
* encrypted.
*/
   public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
   throws IOException {
-final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
+FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
 if (feInfo != null) {
-  // File is encrypted, wrap the stream in a crypto stream.
-  // Currently only one version, so no special logic based on the version #
-  HdfsKMSUtil.getCryptoProtocolVersion(feInfo);
-  final CryptoCodec codec = HdfsKMSUtil.getCryptoCodec(conf, feInfo);
-  final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
-  final CryptoInputStream cryptoIn =
-  new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
-  feInfo.getIV());
+  CryptoInputStream cryptoIn;
+  try (TraceScope ignored = getTracer().newScope("decryptEDEK")) {
+cryptoIn = HdfsKMSUtil.createWrappedInputStream(dfsis,
+