git commit: HADOOP-11017. KMS delegation token secret manager should be able to use zookeeper as store. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 cadba3067 -> 897ced1a2
HADOOP-11017. KMS delegation token secret manager should be able to use
zookeeper as store. (asuresh via tucu)
(cherry picked from commit db890eef3208cc557476fa510f7a253ba22bc68a)
Conflicts:
hadoop-project/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/897ced1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/897ced1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/897ced1a
Branch: refs/heads/branch-2
Commit: 897ced1a20fa74286a5ecadc0b56c95df6322575
Parents: cadba30
Author: Alejandro Abdelnur
Authored: Sat Sep 20 08:20:34 2014 -0700
Committer: Alejandro Abdelnur
Committed: Sat Sep 20 08:24:34 2014 -0700
--
.../util/ZKSignerSecretProvider.java| 2 +
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-common-project/hadoop-common/pom.xml | 13 +
.../AbstractDelegationTokenSecretManager.java | 132 +++-
.../ZKDelegationTokenSecretManager.java | 727 +++
.../DelegationTokenAuthenticationFilter.java| 10 +
.../DelegationTokenAuthenticationHandler.java | 26 +-
.../delegation/web/DelegationTokenManager.java | 76 +-
.../token/delegation/TestDelegationToken.java | 4 +-
.../TestZKDelegationTokenSecretManager.java | 68 ++
.../web/TestDelegationTokenManager.java | 17 +-
hadoop-project/pom.xml | 21 +
12 files changed, 1024 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/897ced1a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index a17b6d4..6c0fbbb 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -197,6 +197,8 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
client = (CuratorFramework) curatorClientObj;
} else {
client = createCuratorClient(config);
+ servletContext.setAttribute(
+ ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE, client);
}
this.tokenValidity = tokenValidity;
shouldDisconnect = Boolean.parseBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/897ced1a/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ed3e050..4b11b1f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -201,6 +201,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10970. Cleanup KMS configuration keys. (wang)
+HADOOP-11017. KMS delegation token secret manager should be able to use
+zookeeper as store. (asuresh via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/897ced1a/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index 4a9fae3..32aea30 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -234,6 +234,19 @@
jsch
+ org.apache.curator
+ curator-test
+ test
+
+
+ org.apache.curator
+ curator-client
+
+
+ org.apache.curator
+ curator-recipes
+
+
com.google.code.findbugs
jsr305
compile
http://git-wip-us.apache.org/repos/asf/hadoop/blob/897ced1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index b9e26b5..f5e7bc9 100
git commit: HADOOP-11017. KMS delegation token secret manager should be able to use zookeeper as store. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk f85cc14eb -> db890eef3
HADOOP-11017. KMS delegation token secret manager should be able to use
zookeeper as store. (asuresh via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db890eef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db890eef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db890eef
Branch: refs/heads/trunk
Commit: db890eef3208cc557476fa510f7a253ba22bc68a
Parents: f85cc14
Author: Alejandro Abdelnur
Authored: Sat Sep 20 08:20:34 2014 -0700
Committer: Alejandro Abdelnur
Committed: Sat Sep 20 08:21:44 2014 -0700
--
.../util/ZKSignerSecretProvider.java| 2 +
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-common-project/hadoop-common/pom.xml | 13 +
.../AbstractDelegationTokenSecretManager.java | 132 +++-
.../ZKDelegationTokenSecretManager.java | 727 +++
.../DelegationTokenAuthenticationFilter.java| 10 +
.../DelegationTokenAuthenticationHandler.java | 26 +-
.../delegation/web/DelegationTokenManager.java | 76 +-
.../token/delegation/TestDelegationToken.java | 4 +-
.../TestZKDelegationTokenSecretManager.java | 68 ++
.../web/TestDelegationTokenManager.java | 17 +-
hadoop-project/pom.xml | 10 +
12 files changed, 1013 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index a17b6d4..6c0fbbb 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -197,6 +197,8 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
client = (CuratorFramework) curatorClientObj;
} else {
client = createCuratorClient(config);
+ servletContext.setAttribute(
+ ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE, client);
}
this.tokenValidity = tokenValidity;
shouldDisconnect = Boolean.parseBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 70579c3..2b07f8d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10970. Cleanup KMS configuration keys. (wang)
+HADOOP-11017. KMS delegation token secret manager should be able to use
+zookeeper as store. (asuresh via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index 0183e29..32e9525 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -219,6 +219,19 @@
jsch
+ org.apache.curator
+ curator-test
+ test
+
+
+ org.apache.curator
+ curator-client
+
+
+ org.apache.curator
+ curator-recipes
+
+
com.google.code.findbugs
jsr305
compile
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db890eef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index b9e26b5..f5e7bc9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegat
git commit: KMS: Support for multiple Kerberos principals. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 372bf5407 -> 22f4ef4fa
KMS: Support for multiple Kerberos principals. (tucu)
(cherry picked from commit fad4cd85b313a1d2378adcf03cad67e946a12cd5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22f4ef4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22f4ef4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22f4ef4f
Branch: refs/heads/branch-2
Commit: 22f4ef4fa9c3820797eed050d48a2780ddfa659a
Parents: 372bf54
Author: Alejandro Abdelnur
Authored: Thu Sep 18 16:03:38 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 18 16:04:18 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../crypto/key/kms/KMSClientProvider.java | 3 +++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +++-
3 files changed, 30 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f4ef4f/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 03a73e1..b325980 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -499,6 +499,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11105. MetricsSystemImpl could leak memory in registered callbacks.
(Chuan Liu via cnauroth)
+KMS: Support for multiple Kerberos principals. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f4ef4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index ea191fc..e3aa1dc 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22f4ef4f/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 8570adf..5ab0bbe 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ <<>>. This means that all KMS instances must have a Kerberos
+ service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Keberos service name with its own hostname. This is
+ required for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+
+hadoop.kms.authentication.kerberos.principal
+*
+
++---+
+
+ <> If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ <<> SAN extension support for details on how to do this).
*** HTTP Authentication Signature
git commit: KMS: Support for multiple Kerberos principals. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 52945a33c -> fad4cd85b
KMS: Support for multiple Kerberos principals. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad4cd85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad4cd85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad4cd85
Branch: refs/heads/trunk
Commit: fad4cd85b313a1d2378adcf03cad67e946a12cd5
Parents: 52945a3
Author: Alejandro Abdelnur
Authored: Thu Sep 18 16:03:38 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 18 16:03:38 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../crypto/key/kms/KMSClientProvider.java | 3 +++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +++-
3 files changed, 30 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2e2d569..f21771b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -834,6 +834,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11105. MetricsSystemImpl could leak memory in registered callbacks.
(Chuan Liu via cnauroth)
+KMS: Support for multiple Kerberos principals. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4cd85/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index b2755a1..cf7a557 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ <<>>. This means that all KMS instances must have a Kerberos
+ service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Keberos service name with its own hostname. This is
+ required for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+
+hadoop.kms.authentication.kerberos.principal
+*
+
++---+
+
+ <> If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ <<> SAN extension support for details on how to do this).
*** HTTP Authentication Signature
git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Repository: hadoop Updated Branches: refs/heads/branch-2 3746b1e90 -> d3efebf4a HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) (cherry picked from commit 123f20d42f6acffcde05392d689acd91a82462db) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3efebf4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3efebf4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3efebf4 Branch: refs/heads/branch-2 Commit: d3efebf4aaf4a8da602c9f134d5b0f9cf0b8b5b7 Parents: 3746b1e Author: Alejandro Abdelnur Authored: Wed Sep 17 14:27:35 2014 -0700 Committer: Alejandro Abdelnur Committed: Wed Sep 17 15:30:56 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 7 files changed, 373 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e5a914e..6661bfb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -194,6 +194,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) +HADOOP-11016. KMS should support signing cookies with zookeeper secret +manager. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 37dcb2c..9de5c45 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ metrics-core compile + + org.apache.curator + curator-test + test + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ + + + +hadoop.kms.authentication.signer.secret.provider +random + + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. + + + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.path +/hadoop-kms/hadoop-auth-signature-secret + + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string +#HOSTNAME#:#PORT#,... + + The Zookeeper connection string, a list of hostnames and port comma + separated. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type +kerberos + + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab +/etc/hadoop/conf/kms.keytab + + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal +kms/#HOSTNAME# + + The Kerberos service principal used to connect to Zookeeper. + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --g
git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Repository: hadoop Updated Branches: refs/heads/trunk f4886111a -> 123f20d42 HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/123f20d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/123f20d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/123f20d4 Branch: refs/heads/trunk Commit: 123f20d42f6acffcde05392d689acd91a82462db Parents: f488611 Author: Alejandro Abdelnur Authored: Wed Sep 17 14:27:35 2014 -0700 Committer: Alejandro Abdelnur Committed: Wed Sep 17 15:29:17 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 7 files changed, 373 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 31c09de..d2671c3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -532,6 +532,9 @@ Release 2.6.0 - UNRELEASED HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) +HADOOP-11016. KMS should support signing cookies with zookeeper secret +manager. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 2c225cb..e6b21aa 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ metrics-core compile + + org.apache.curator + curator-test + test + http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ + + + +hadoop.kms.authentication.signer.secret.provider +random + + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. + + + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.path +/hadoop-kms/hadoop-auth-signature-secret + + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string +#HOSTNAME#:#PORT#,... + + The Zookeeper connection string, a list of hostnames and port comma + separated. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type +kerberos + + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab +/etc/hadoop/conf/kms.keytab + + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal +kms/#HOSTNAME# + + The Kerberos service principal used to connect to Zookeeper. + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/h
git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1c847fdd6 -> 6857c291a
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
(cherry picked from commit e4ddb6da15420d5c13ec7ec99fed1e44b32290b0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6857c291
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6857c291
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6857c291
Branch: refs/heads/branch-2
Commit: 6857c291af05350064336ba12c121c7fada27a5d
Parents: 1c847fd
Author: Alejandro Abdelnur
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:08:25 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6857c291/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0fad37d..40b0045 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -489,6 +489,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6857c291/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapper {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
[2/2] git commit: Revert "HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)"
Revert "HADOOP-11016. KMS should support signing cookies with zookeeper secret
manager. (tucu)"
This reverts commit 0a495bef5cd675dce4c928cb5331588bb198accf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a7671d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a7671d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a7671d7
Branch: refs/heads/trunk
Commit: 8a7671d7539bff0566cb87f2b347f71bcf148977
Parents: 3f8f860
Author: Alejandro Abdelnur
Authored: Wed Sep 17 11:11:33 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:11:33 2014 -0700
--
hadoop-common-project/hadoop-kms/pom.xml| 5 -
.../hadoop-kms/src/main/conf/kms-site.xml | 57 --
.../key/kms/server/KMSAuthenticationFilter.java | 7 +-
.../hadoop-kms/src/site/apt/index.apt.vm| 161 -
.../hadoop/crypto/key/kms/server/TestKMS.java | 5 +-
.../crypto/key/kms/server/TestKMSWithZK.java| 179 ---
6 files changed, 44 insertions(+), 370 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index e6b21aa..2c225cb 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -187,11 +187,6 @@
metrics-core
compile
-
- org.apache.curator
- curator-test
- test
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index f55ce5f..20896fc 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -68,61 +68,4 @@
-
-
-
-hadoop.kms.authentication.signer.secret.provider
-random
-
- Indicates how the secret to sign the authentication cookies will be
- stored. Options are 'random' (default), 'string' and 'zookeeper'.
- If using a setup with multiple KMS instances, 'zookeeper' should be used.
-
-
-
-
-
-
-
hadoop.kms.authentication.signer.secret.provider.zookeeper.path
-/hadoop-kms/hadoop-auth-signature-secret
-
- The Zookeeper ZNode path where the KMS instances will store and retrieve
- the secret from.
-
-
-
-
-
hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string
-#HOSTNAME#:#PORT#,...
-
- The Zookeeper connection string, a list of hostnames and port comma
- separated.
-
-
-
-
-
hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type
-kerberos
-
- The Zookeeper authentication type, 'none' or 'sasl' (Kerberos).
-
-
-
-
-
hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab
-/etc/hadoop/conf/kms.keytab
-
- The absolute path for the Kerberos keytab with the credentials to
- connect to Zookeeper.
-
-
-
-
-
hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal
-kms/#HOSTNAME#
-
- The Kerberos service principal used to connect to Zookeeper.
-
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
index 79652f3..4df6db5 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -46,8 +46,7 @@ import java.util.Properties;
@InterfaceAudience.Private
public class KMSAuthenticationFilter
extends DelegationTokenAuthenticationFilter {
-
- public static final String CONFIG_PREFIX = KMSConfiguration.CONFIG_PREFIX +
+ private static final String CONF_PREFIX = KMSConfiguration.CONFIG_PREFIX +
"authentication.";
@Override
@@ -57,9 +56,9 @@ public class KMSAuthenticationFilter
Con
[1/2] git commit: Revert "HADOOP-10982"
Repository: hadoop
Updated Branches:
refs/heads/trunk d9a86031a -> 8a7671d75
Revert "HADOOP-10982"
This reverts commit d9a86031a077184d429dd5463e7da156df112011.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f8f860c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f8f860c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f8f860c
Branch: refs/heads/trunk
Commit: 3f8f860cc65e179dd5766fea4d21cf30fa4b96e3
Parents: d9a8603
Author: Alejandro Abdelnur
Authored: Wed Sep 17 11:11:15 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:11:15 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 --
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 11 insertions(+), 72 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a97463a..899b6c4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,7 +45,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -401,8 +400,6 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
-} catch (UndeclaredThrowableException ex) {
- throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 682f479..5fded92 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,31 +602,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- When KMS instances are behind a load-balancer or VIP, clients will use the
- hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
- URL is used to construct the Kerberos service name of the server,
- <<>>. This means that all KMS instances must have have a
- Kerberos service name with the load-balancer or VIP hostname.
-
- In order to be able to access directly a specific KMS instance, the KMS
- instance must also have Kebero service name with its own hostname. This is
- require for monitoring and admin purposes.
-
- Both Kerberos service principal credentials (for the load-balancer/VIP
- hostname and for the actual KMS instance hostname) must be in the keytab file
- configured for authentication. And the principal name specified in the
- configuration must be '*'. For example:
-
-+---+
-
-hadoop.kms.authentication.kerberos.principal
-*
-
-+---+
-
- <> If using HTTPS, the SSL certificate used by the KMS instance must
- be configured to support multiple hostnames (see Java 7
- <<> SAN extension support for details on how to do this).
+ TBD
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 42afe19..cdb3c7f 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.Secur
[1/3] git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c0c7e6fab -> d9a86031a
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4ddb6da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4ddb6da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4ddb6da
Branch: refs/heads/trunk
Commit: e4ddb6da15420d5c13ec7ec99fed1e44b32290b0
Parents: c0c7e6f
Author: Alejandro Abdelnur
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:07:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f0fcab5..a1dca66 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -824,6 +824,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapper {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
[2/3] git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
HADOOP-11016. KMS should support signing cookies with zookeeper secret manager.
(tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a495bef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a495bef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a495bef
Branch: refs/heads/trunk
Commit: 0a495bef5cd675dce4c928cb5331588bb198accf
Parents: e4ddb6d
Author: Alejandro Abdelnur
Authored: Tue Sep 16 21:21:17 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:08:00 2014 -0700
--
hadoop-common-project/hadoop-kms/pom.xml| 5 +
.../hadoop-kms/src/main/conf/kms-site.xml | 57 ++
.../key/kms/server/KMSAuthenticationFilter.java | 7 +-
.../hadoop-kms/src/site/apt/index.apt.vm| 161 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 5 +-
.../crypto/key/kms/server/TestKMSWithZK.java| 179 +++
6 files changed, 370 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 2c225cb..e6b21aa 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -187,6 +187,11 @@
metrics-core
compile
+
+ org.apache.curator
+ curator-test
+ test
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index 20896fc..f55ce5f 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -68,4 +68,61 @@
+
+
+
+hadoop.kms.authentication.signer.secret.provider
+random
+
+ Indicates how the secret to sign the authentication cookies will be
+ stored. Options are 'random' (default), 'string' and 'zookeeper'.
+ If using a setup with multiple KMS instances, 'zookeeper' should be used.
+
+
+
+
+
+
+
hadoop.kms.authentication.signer.secret.provider.zookeeper.path
+/hadoop-kms/hadoop-auth-signature-secret
+
+ The Zookeeper ZNode path where the KMS instances will store and retrieve
+ the secret from.
+
+
+
+
+
hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string
+#HOSTNAME#:#PORT#,...
+
+ The Zookeeper connection string, a list of hostnames and port comma
+ separated.
+
+
+
+
+
hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type
+kerberos
+
+ The Zookeeper authentication type, 'none' or 'sasl' (Kerberos).
+
+
+
+
+
hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab
+/etc/hadoop/conf/kms.keytab
+
+ The absolute path for the Kerberos keytab with the credentials to
+ connect to Zookeeper.
+
+
+
+
+
hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal
+kms/#HOSTNAME#
+
+ The Kerberos service principal used to connect to Zookeeper.
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
index 4df6db5..79652f3 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
@@ -46,7 +46,8 @@ import java.util.Properties;
@InterfaceAudience.Private
public class KMSAuthenticationFilter
extends DelegationTokenAuthenticationFilter {
- private static final String CONF_PREFIX = KMSConfiguration.CONFIG_PREFIX +
+
+ public static final String CONFIG_PREFIX = KMSConfiguration.CONFIG_PREFIX +
"authentication.";
@Override
@@ -56,9 +57,9 @@ public class KMSAuthenticationFilter
Configuration conf = KMSWebApp.getConfiguration();
for (Map.Entry entry : conf) {
[3/3] git commit: HADOOP-10982
HADOOP-10982
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a86031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a86031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a86031
Branch: refs/heads/trunk
Commit: d9a86031a077184d429dd5463e7da156df112011
Parents: 0a495be
Author: Alejandro Abdelnur
Authored: Tue Sep 16 23:07:01 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 17 11:08:00 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 ++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 72 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 5fded92..682f479 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ <<>>. This means that all KMS instances must have have a
+ Kerberos service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Kebero service name with its own hostname. This is
+ require for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+
+hadoop.kms.authentication.kerberos.principal
+*
+
++---+
+
+ <> If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ <<> SAN extension support for details on how to do this).
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index cdb3c7f..42afe19 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import
org.apache.hadoop.security.authentication.client.AuthenticationException;
imp
git commit: HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if -Pnative is used. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 75bd79231 -> 1c847fdd6
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if
-Pnative is used. (asuresh via tucu)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c847fdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c847fdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c847fdd
Branch: refs/heads/branch-2
Commit: 1c847fdd61414f7f564de2cc477621edac8164b5
Parents: 75bd792
Author: Alejandro Abdelnur
Authored: Tue Sep 16 23:36:10 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:37:21 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 3 +++
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 18 --
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 11 +++
4 files changed, 33 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d6b05f7..0fad37d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -486,6 +486,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
(clamb via tucu)
+HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
+only if -Pnative is used. (asuresh via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index cb6bafa..4a9fae3 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -390,6 +390,7 @@
${startKdc}
${kdc.resource.dir}
+${runningWithNative}
@@ -528,6 +529,7 @@
false
+true
@@ -647,6 +649,7 @@
false
+true
true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 298f4ef..79987ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -59,7 +59,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
+ LOG.warn("Skipping since test was not run with -Pnative flag");
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn("Skipping test since openSSL library not loaded");
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
@@ -68,7 +75,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
+ LOG.warn("Skipping since test was not run with -Pnative flag");
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn("Skipping test since openSSL library not loaded");
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass);
cryptoCodecTest(conf, seed, co
git commit: HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if -Pnative is used. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8cf1052be -> c0c7e6fab
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if
-Pnative is used. (asuresh via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0c7e6fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0c7e6fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0c7e6fa
Branch: refs/heads/trunk
Commit: c0c7e6fabd573df85791d7ec4c536fd48280883f
Parents: 8cf1052
Author: Alejandro Abdelnur
Authored: Tue Sep 16 23:36:10 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:36:36 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 3 +++
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 18 --
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 +++
4 files changed, 29 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 11151f0..f0fcab5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -821,6 +821,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
(clamb via tucu)
+HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
+only if -Pnative is used. (asuresh via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index ae495be..0183e29 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -375,6 +375,7 @@
${startKdc}
${kdc.resource.dir}
+${runningWithNative}
@@ -507,6 +508,7 @@
false
+true
@@ -626,6 +628,7 @@
false
+true
true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 298f4ef..79987ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -59,7 +59,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
+ LOG.warn("Skipping since test was not run with -Pnative flag");
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn("Skipping test since openSSL library not loaded");
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
@@ -68,7 +75,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
+ LOG.warn("Skipping since test was not run with -Pnative flag");
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn("Skipping test since openSSL library not loaded");
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass);
cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass);
http://git-wip-u
git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk e14e71d5f -> 8cf1052be
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cf1052b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cf1052b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cf1052b
Branch: refs/heads/trunk
Commit: 8cf1052beb7cab68be1a6319c0a4d7e1c790d58a
Parents: e14e71d
Author: Alejandro Abdelnur
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9324acd..11151f0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -818,6 +818,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index c76ca3b..d70f2a6 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in <<>> using the
+ Each proxyuser must be configured in <<>> using the
following properties:
+---+
-hadoop.kms.proxyusers.#USER#.users
+hadoop.kms.proxyuser.#USER#.users
*
-hadoop.kms.proxyusers.#USER#.groups
+hadoop.kms.proxyuser.#USER#.groups
*
-hadoop.kms.proxyusers.#USER#.hosts
+hadoop.kms.proxyuser.#USER#.hosts
*
+---+
git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 94a1e68aa -> 75bd79231
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75bd7923
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75bd7923
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75bd7923
Branch: refs/heads/branch-2
Commit: 75bd79231ca30cb7a16107101c175c5b6fa06f56
Parents: 94a1e68
Author: Alejandro Abdelnur
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:21:17 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75bd7923/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 939af25..d6b05f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -483,6 +483,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75bd7923/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index be6c8f1..02ca1c5 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in <<>> using the
+ Each proxyuser must be configured in <<>> using the
following properties:
+---+
-hadoop.kms.proxyusers.#USER#.users
+hadoop.kms.proxyuser.#USER#.users
*
-hadoop.kms.proxyusers.#USER#.groups
+hadoop.kms.proxyuser.#USER#.groups
*
-hadoop.kms.proxyusers.#USER#.hosts
+hadoop.kms.proxyuser.#USER#.hosts
*
+---+
git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c6b9768b3 -> 94a1e68aa
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94a1e68a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94a1e68a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94a1e68a
Branch: refs/heads/branch-2
Commit: 94a1e68aa5aa3ea633b3af7b09aa2b9012498101
Parents: c6b9768
Author: Alejandro Abdelnur
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:21:17 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0ec1264..939af25 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -480,6 +480,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 5d3281c..f800689 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 9893515..0b202ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
@@ -121,7 +12
git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0e7d1dbf9 -> e14e71d5f
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e14e71d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e14e71d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e14e71d5
Branch: refs/heads/trunk
Commit: e14e71d5feff961b681d828b00e6f12cb197ebf5
Parents: 0e7d1db
Author: Alejandro Abdelnur
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bf9d4b..9324acd 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -815,6 +815,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index fed7e9e..968e341 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 70ec6fe..62e3310 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
@@ -121,7 +121,7 @@ pub
git commit: HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9be338911 -> 5d897026e
HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d897026
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d897026
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d897026
Branch: refs/heads/branch-2
Commit: 5d897026e426737d792ef7922052872e869d6785
Parents: 9be3389
Author: Alejandro Abdelnur
Authored: Tue Sep 16 12:39:17 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 14:37:04 2014 -0700
--
hadoop-common-project/hadoop-kms/pom.xml| 25 +++-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 47 +--
.../test/resources/mini-kms-acls-default.xml| 135 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 13 ++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 56
hadoop-project/pom.xml | 14 ++
8 files changed, 289 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d897026/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 481f80e..37dcb2c 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -238,7 +238,7 @@
default-war
-package
+prepare-package
war
@@ -252,6 +252,29 @@
+org.apache.maven.plugins
+maven-jar-plugin
+
+
+prepare-jar
+prepare-package
+
+ jar
+
+
+ classes
+
+
+
+prepare-test-jar
+prepare-package
+
+ test-jar
+
+
+
+
+
org.codehaus.mojo
findbugs-maven-plugin
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d897026/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 195eee8..f64dcf0 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.fs.Path;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@@ -26,7 +28,10 @@ import org.mortbay.jetty.security.SslSocketConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Writer;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -34,6 +39,7 @@ import java.net.ServerSocket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.UUID;
public class MiniKMS {
@@ -140,13 +146,15 @@ public class MiniKMS {
}
public void start() throws Exception {
+ClassLoader cl = Thread.currentThread().getContextClassLoader();
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
if (!aclsFile.exists()) {
- Configuration acls = new Configuration(false);
- Writer writer = new FileWriter(aclsFile);
- acls.writeXml(writer);
- writer.close();
+ InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
+ OutputStream os = new FileOutputStream(aclsFile);
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
}
File coreFile = new File(kmsConfDir, "core-site.xml");
if (!coreFile.exists()) {
@@ -161,19 +169,42 @@ public class MiniKMS {
kms.set("hadoop.security.key.provider.path",
"jceks://file@" + new Path
git commit: HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk ffdb7eb3b -> 3e85f5b60
HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e85f5b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e85f5b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e85f5b6
Branch: refs/heads/trunk
Commit: 3e85f5b605b9ccee54aba7b4a683f81734571d60
Parents: ffdb7eb
Author: Alejandro Abdelnur
Authored: Tue Sep 16 12:39:17 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 16 14:36:07 2014 -0700
--
hadoop-common-project/hadoop-kms/pom.xml| 25 +++-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 47 +--
.../test/resources/mini-kms-acls-default.xml| 135 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 13 ++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 56
hadoop-project/pom.xml | 14 ++
8 files changed, 289 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e85f5b6/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 629ffda..2c225cb 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -238,7 +238,7 @@
default-war
-package
+prepare-package
war
@@ -252,6 +252,29 @@
+org.apache.maven.plugins
+maven-jar-plugin
+
+
+prepare-jar
+prepare-package
+
+ jar
+
+
+ classes
+
+
+
+prepare-test-jar
+prepare-package
+
+ test-jar
+
+
+
+
+
org.codehaus.mojo
findbugs-maven-plugin
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e85f5b6/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 195eee8..f64dcf0 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.fs.Path;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@@ -26,7 +28,10 @@ import org.mortbay.jetty.security.SslSocketConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Writer;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -34,6 +39,7 @@ import java.net.ServerSocket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.UUID;
public class MiniKMS {
@@ -140,13 +146,15 @@ public class MiniKMS {
}
public void start() throws Exception {
+ClassLoader cl = Thread.currentThread().getContextClassLoader();
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
if (!aclsFile.exists()) {
- Configuration acls = new Configuration(false);
- Writer writer = new FileWriter(aclsFile);
- acls.writeXml(writer);
- writer.close();
+ InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
+ OutputStream os = new FileOutputStream(aclsFile);
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
}
File coreFile = new File(kmsConfDir, "core-site.xml");
if (!coreFile.exists()) {
@@ -161,19 +169,42 @@ public class MiniKMS {
kms.set("hadoop.security.key.provider.path",
"jceks://file@" + new Path
git commit: HADOOP-10868. Addendum
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e59f6771e -> 1023196ce
HADOOP-10868. Addendum
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1023196c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1023196c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1023196c
Branch: refs/heads/branch-2
Commit: 1023196ceaa600f92f328cfe67a8bccac3445a64
Parents: e59f677
Author: Alejandro Abdelnur
Authored: Mon Sep 15 19:39:12 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 19:39:12 2014 -0700
--
.../security/authentication/util/ZKSignerSecretProvider.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1023196c/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 45d4d65..a17b6d4 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -139,6 +139,9 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE =
CONFIG_PREFIX + "curator.client";
+ private static final String JAAS_LOGIN_ENTRY_NAME =
+ "ZKSignerSecretProviderClient";
+
private static Logger LOG = LoggerFactory.getLogger(
ZKSignerSecretProvider.class);
private String path;
@@ -384,7 +387,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
+ "and using 'sasl' ACLs");
String principal = setJaasConfiguration(config);
System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
- "ZKSignerSecretProviderClient");
+ JAAS_LOGIN_ENTRY_NAME);
System.setProperty("zookeeper.authProvider.1",
"org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
aclProvider = new SASLOwnerACLProvider(principal);
@@ -417,7 +420,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
// This is equivalent to writing a jaas.conf file and setting the system
// property, "java.security.auth.login.config", to point to it
JaasConfiguration jConf =
-new JaasConfiguration("Client", principal, keytabFile);
+new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal,
keytabFile);
Configuration.setConfiguration(jConf);
return principal.split("[/@]")[0];
}
git commit: HADOOP-10868. Addendum
Repository: hadoop
Updated Branches:
refs/heads/trunk 932ae036a -> 7e08c0f23
HADOOP-10868. Addendum
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e08c0f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e08c0f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e08c0f2
Branch: refs/heads/trunk
Commit: 7e08c0f23f58aa143f0997f2472e8051175142e9
Parents: 932ae03
Author: Alejandro Abdelnur
Authored: Mon Sep 15 19:39:27 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 19:39:27 2014 -0700
--
.../security/authentication/util/ZKSignerSecretProvider.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e08c0f2/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 45d4d65..a17b6d4 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -139,6 +139,9 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE =
CONFIG_PREFIX + "curator.client";
+ private static final String JAAS_LOGIN_ENTRY_NAME =
+ "ZKSignerSecretProviderClient";
+
private static Logger LOG = LoggerFactory.getLogger(
ZKSignerSecretProvider.class);
private String path;
@@ -384,7 +387,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
+ "and using 'sasl' ACLs");
String principal = setJaasConfiguration(config);
System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
- "ZKSignerSecretProviderClient");
+ JAAS_LOGIN_ENTRY_NAME);
System.setProperty("zookeeper.authProvider.1",
"org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
aclProvider = new SASLOwnerACLProvider(principal);
@@ -417,7 +420,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
// This is equivalent to writing a jaas.conf file and setting the system
// property, "java.security.auth.login.config", to point to it
JaasConfiguration jConf =
-new JaasConfiguration("Client", principal, keytabFile);
+new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal,
keytabFile);
Configuration.setConfiguration(jConf);
return principal.split("[/@]")[0];
}
[2/2] git commit: HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
HADOOP-10868. AuthenticationFilter should support externalizing the secret for
signing and provide rotation support. (rkanter via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e59f6771
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e59f6771
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e59f6771
Branch: refs/heads/branch-2
Commit: e59f6771e89ded737cc91698763a02f6ebf23c61
Parents: f80b10e
Author: Alejandro Abdelnur
Authored: Mon Sep 15 17:10:43 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 17:10:43 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java| 152 --
.../util/RandomSignerSecretProvider.java| 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java| 15 +-
.../util/ZKSignerSecretProvider.java| 503 +++
.../src/site/apt/Configuration.apt.vm | 148 +-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java| 117 -
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java| 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java| 9 +-
.../util/TestZKSignerSecretProvider.java| 270 ++
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-project/pom.xml | 11 +
19 files changed, 1259 insertions(+), 97 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index 20304e1..1da98dc 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -135,6 +135,19 @@
+
+ org.apache.zookeeper
+ zookeeper
+
+
+ org.apache.curator
+ curator-framework
+
+
+ org.apache.curator
+ curator-test
+ test
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 9330444..47cf54c 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -22,6 +22,7 @@ import
org.apache.hadoop.security.authentication.util.SignerException;
import
org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import
org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
+import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import java.util.*;
/**
* The {@link AuthenticationFilter} enables protecting web application
resources with different (pluggable)
- * authentication mechanisms.
+ * authentication mechanisms and signer secret providers.
*
* Out of the box it provides 2 authentication mechanisms: Pseudo and Kerberos
SPNEGO.
*
@@ -60,10 +61,13 @@ import java.util.*;
* [#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the
* {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link
KerberosAuthenticationHandler}, otherwise
* the full class name of the {@link AuthenticationHandler} must be
specified.
- * [#PREFIX#.]signature.secret: the secret used to sign the HTTP cookie
value. The default value is a random
- * value. Unless multiple webapp instances need to share the secret the random
value is adequate.
- * [#PREFIX#.]token.validity: time -in seconds- that the generated token
is valid before a
- * new authentication is triggered, default value is 3600
seconds.
+ * [#PREFIX#.]signature.secret: when signer.se
[1/2] HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
secretB3 = Long.toString(rand.nextLong()).getBytes();
+ZKSignerSecretProvider secretProviderA = new ZKSignerSecretProvider(seedA);
+ZKSignerSecretProvider secretProviderB = new ZKSignerSecretProvider(seedB);
+Properties config = new Properties();
+config.setProperty(
+ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING,
+zkServer.getConnectString());
+config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH,
+"/secret");
+try {
+ secretProviderA.init(config, getDummyServletContext(),
rolloverFrequency);
+
+ byte[] currentSecretA = secretProviderA.getCurrentSecret();
+ byte[][] allSecretsA = secretProviderA.getAllSecrets();
+ Assert.assertArrayEquals(secretA1, currentSecretA);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA1, allSecretsA[0]);
+ Assert.assertNull(allSecretsA[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecretA = secretProviderA.getCurrentSecret();
+ allSecretsA = secretProviderA.getAllSecrets();
+ Assert.assertArrayEquals(secretA2, currentSecretA);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA2, allSecretsA[0]);
+ Assert.assertArrayEquals(secretA1, allSecretsA[1]);
+ Thread.sleep((rolloverFrequency / 5));
+
+ secretProviderB.init(config, getDummyServletContext(),
rolloverFrequency);
+
+ byte[] currentSecretB = secretProviderB.getCurrentSecret();
+ byte[][] allSecretsB = secretProviderB.getAllSecrets();
+ Assert.assertArrayEquals(secretA2, currentSecretB);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA2, allSecretsB[0]);
+ Assert.assertArrayEquals(secretA1, allSecretsB[1]);
+ Thread.sleep((rolloverFrequency));
+
+ currentSecretA = secretProviderA.getCurrentSecret();
+ allSecretsA = secretProviderA.getAllSecrets();
+ currentSecretB = secretProviderB.getCurrentSecret();
+ allSecretsB = secretProviderB.getAllSecrets();
+ Assert.assertArrayEquals(currentSecretA, currentSecretB);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertEquals(2, allSecretsB.length);
+ Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]);
+ Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]);
+ if (Arrays.equals(secretA3, currentSecretA)) {
+Assert.assertArrayEquals(secretA3, allSecretsA[0]);
+ } else if (Arrays.equals(secretB3, currentSecretB)) {
+Assert.assertArrayEquals(secretB3, allSecretsA[0]);
+ } else {
+Assert.fail("It appears that they all agreed on the same secret, but "
++ "not one of the secrets they were supposed to");
+ }
+} finally {
+ secretProviderB.destroy();
+ secretProviderA.destroy();
+}
+ }
+
+ private ServletContext getDummyServletContext() {
+ServletContext servletContext = Mockito.mock(ServletContext.class);
+Mockito.when(servletContext.getAttribute(ZKSignerSecretProvider
+.ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE))
+.thenReturn(null);
+return servletContext;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index fddd86d..6a82d61 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -189,6 +189,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11091. Eliminate old configuration parameter names from s3a (David
S. Wang via Colin Patrick McCabe)
+HADOOP-10868. AuthenticationFilter should support externalizing the
+secret for signing and provide rotation support. (rkanter via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index c6c0d19..763d168 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -66,6 +66,8 @@ import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import com.google.common.collect.Maps;
+import java.util.Properties;
+import org.apache.hadoop.securi
[1/2] HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
secretB3 = Long.toString(rand.nextLong()).getBytes();
+ZKSignerSecretProvider secretProviderA = new ZKSignerSecretProvider(seedA);
+ZKSignerSecretProvider secretProviderB = new ZKSignerSecretProvider(seedB);
+Properties config = new Properties();
+config.setProperty(
+ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING,
+zkServer.getConnectString());
+config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH,
+"/secret");
+try {
+ secretProviderA.init(config, getDummyServletContext(),
rolloverFrequency);
+
+ byte[] currentSecretA = secretProviderA.getCurrentSecret();
+ byte[][] allSecretsA = secretProviderA.getAllSecrets();
+ Assert.assertArrayEquals(secretA1, currentSecretA);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA1, allSecretsA[0]);
+ Assert.assertNull(allSecretsA[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecretA = secretProviderA.getCurrentSecret();
+ allSecretsA = secretProviderA.getAllSecrets();
+ Assert.assertArrayEquals(secretA2, currentSecretA);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA2, allSecretsA[0]);
+ Assert.assertArrayEquals(secretA1, allSecretsA[1]);
+ Thread.sleep((rolloverFrequency / 5));
+
+ secretProviderB.init(config, getDummyServletContext(),
rolloverFrequency);
+
+ byte[] currentSecretB = secretProviderB.getCurrentSecret();
+ byte[][] allSecretsB = secretProviderB.getAllSecrets();
+ Assert.assertArrayEquals(secretA2, currentSecretB);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertArrayEquals(secretA2, allSecretsB[0]);
+ Assert.assertArrayEquals(secretA1, allSecretsB[1]);
+ Thread.sleep((rolloverFrequency));
+
+ currentSecretA = secretProviderA.getCurrentSecret();
+ allSecretsA = secretProviderA.getAllSecrets();
+ currentSecretB = secretProviderB.getCurrentSecret();
+ allSecretsB = secretProviderB.getAllSecrets();
+ Assert.assertArrayEquals(currentSecretA, currentSecretB);
+ Assert.assertEquals(2, allSecretsA.length);
+ Assert.assertEquals(2, allSecretsB.length);
+ Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]);
+ Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]);
+ if (Arrays.equals(secretA3, currentSecretA)) {
+Assert.assertArrayEquals(secretA3, allSecretsA[0]);
+ } else if (Arrays.equals(secretB3, currentSecretB)) {
+Assert.assertArrayEquals(secretB3, allSecretsA[0]);
+ } else {
+Assert.fail("It appears that they all agreed on the same secret, but "
++ "not one of the secrets they were supposed to");
+ }
+} finally {
+ secretProviderB.destroy();
+ secretProviderA.destroy();
+}
+ }
+
+ private ServletContext getDummyServletContext() {
+ServletContext servletContext = Mockito.mock(ServletContext.class);
+Mockito.when(servletContext.getAttribute(ZKSignerSecretProvider
+.ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE))
+.thenReturn(null);
+return servletContext;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 89bce4d..2d906f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -520,6 +520,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11091. Eliminate old configuration parameter names from s3a (David
S. Wang via Colin Patrick McCabe)
+HADOOP-10868. AuthenticationFilter should support externalizing the
+secret for signing and provide rotation support. (rkanter via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index c6c0d19..763d168 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -66,6 +66,8 @@ import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import com.google.common.collect.Maps;
+import java.util.Properties;
+import org.apache.hadoop.security.authe
[2/2] git commit: HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
HADOOP-10868. AuthenticationFilter should support externalizing the secret for
signing and provide rotation support. (rkanter via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/932ae036
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/932ae036
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/932ae036
Branch: refs/heads/trunk
Commit: 932ae036acb96634c5dd435d57ba02ce4d5e8918
Parents: 0ac760a
Author: Alejandro Abdelnur
Authored: Mon Sep 15 17:05:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 17:05:42 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java| 152 --
.../util/RandomSignerSecretProvider.java| 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java| 15 +-
.../util/ZKSignerSecretProvider.java| 503 +++
.../src/site/apt/Configuration.apt.vm | 148 +-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java| 117 -
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java| 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java| 9 +-
.../util/TestZKSignerSecretProvider.java| 270 ++
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-project/pom.xml | 11 +
19 files changed, 1259 insertions(+), 97 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index 564518c..5f7d774 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -130,6 +130,19 @@
+
+ org.apache.zookeeper
+ zookeeper
+
+
+ org.apache.curator
+ curator-framework
+
+
+ org.apache.curator
+ curator-test
+ test
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 9330444..47cf54c 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -22,6 +22,7 @@ import
org.apache.hadoop.security.authentication.util.SignerException;
import
org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import
org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
+import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import java.util.*;
/**
* The {@link AuthenticationFilter} enables protecting web application
resources with different (pluggable)
- * authentication mechanisms.
+ * authentication mechanisms and signer secret providers.
*
* Out of the box it provides 2 authentication mechanisms: Pseudo and Kerberos
SPNEGO.
*
@@ -60,10 +61,13 @@ import java.util.*;
* [#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the
* {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link
KerberosAuthenticationHandler}, otherwise
* the full class name of the {@link AuthenticationHandler} must be
specified.
- * [#PREFIX#.]signature.secret: the secret used to sign the HTTP cookie
value. The default value is a random
- * value. Unless multiple webapp instances need to share the secret the random
value is adequate.
- * [#PREFIX#.]token.validity: time -in seconds- that the generated token
is valid before a
- * new authentication is triggered, default value is 3600
seconds.
+ * [#PREFIX#.]signature.secret: when signer.secre
git commit: HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is case sensitive. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b4ab7aa11 -> 2924de58c
HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is case
sensitive. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2924de58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2924de58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2924de58
Branch: refs/heads/branch-2
Commit: 2924de58ce8cdb59dc0f492458db5209e972abd7
Parents: b4ab7aa
Author: Alejandro Abdelnur
Authored: Thu Sep 11 13:53:31 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 11 13:54:59 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../web/DelegationTokenAuthenticationFilter.java | 3 ++-
.../delegation/web/TestWebDelegationToken.java | 17 +
3 files changed, 22 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2924de58/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8d03b01..f228d7e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -457,6 +457,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when
value is NaN (Mit Desai via jlowe)
+HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is
+case sensitive. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2924de58/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index 37474e9..64a5622 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -188,7 +188,8 @@ public class DelegationTokenAuthenticationFilter
UTF8_CHARSET);
if (list != null) {
for (NameValuePair nv : list) {
-if (DelegationTokenAuthenticatedURL.DO_AS.equals(nv.getName())) {
+if (DelegationTokenAuthenticatedURL.DO_AS.
+equalsIgnoreCase(nv.getName())) {
return nv.getValue();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2924de58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
index 118abff..189a334 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
@@ -795,6 +795,23 @@ public class TestWebDelegationToken {
jetty.start();
final URL url = new URL(getJettyURL() + "/foo/bar");
+ // proxyuser using raw HTTP, verifying doAs is case insensitive
+ String strUrl = String.format("%s?user.name=%s&doas=%s",
+ url.toExternalForm(), FOO_USER, OK_USER);
+ HttpURLConnection conn =
+ (HttpURLConnection) new URL(strUrl).openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ List ret = IOUtils.readLines(conn.getInputStream());
+ Assert.assertEquals(1, ret.size());
+ Assert.assertEquals(OK_USER, ret.get(0));
+ strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(),
+ FOO_USER, OK_USER);
+ conn = (HttpURLConnection) new URL(strUrl).openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ ret = IOUtils.readLines(conn.getInputStream());
+ Assert.assertEquals(1, ret.size());
+
git commit: HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is case sensitive. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 581176cdc -> c656d7d6e
HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is case
sensitive. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c656d7d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c656d7d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c656d7d6
Branch: refs/heads/trunk
Commit: c656d7d6e53436bf082f76e5988e39d8e18ed64f
Parents: 581176c
Author: Alejandro Abdelnur
Authored: Thu Sep 11 13:53:31 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 11 13:53:31 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../web/DelegationTokenAuthenticationFilter.java | 3 ++-
.../delegation/web/TestWebDelegationToken.java | 17 +
3 files changed, 22 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c656d7d6/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bf92ec..f7cbc8c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -787,6 +787,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when
value is NaN (Mit Desai via jlowe)
+HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is
+case sensitive. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c656d7d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index 37474e9..64a5622 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -188,7 +188,8 @@ public class DelegationTokenAuthenticationFilter
UTF8_CHARSET);
if (list != null) {
for (NameValuePair nv : list) {
-if (DelegationTokenAuthenticatedURL.DO_AS.equals(nv.getName())) {
+if (DelegationTokenAuthenticatedURL.DO_AS.
+equalsIgnoreCase(nv.getName())) {
return nv.getValue();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c656d7d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
index 118abff..189a334 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
@@ -795,6 +795,23 @@ public class TestWebDelegationToken {
jetty.start();
final URL url = new URL(getJettyURL() + "/foo/bar");
+ // proxyuser using raw HTTP, verifying doAs is case insensitive
+ String strUrl = String.format("%s?user.name=%s&doas=%s",
+ url.toExternalForm(), FOO_USER, OK_USER);
+ HttpURLConnection conn =
+ (HttpURLConnection) new URL(strUrl).openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ List ret = IOUtils.readLines(conn.getInputStream());
+ Assert.assertEquals(1, ret.size());
+ Assert.assertEquals(OK_USER, ret.get(0));
+ strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(),
+ FOO_USER, OK_USER);
+ conn = (HttpURLConnection) new URL(strUrl).openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ ret = IOUtils.readLines(conn.getInputStream());
+ Assert.assertEquals(1, ret.size());
+
git commit: HADOOP-10758. KMS: add ACLs on per key basis. (tucu)
Repository: hadoop Updated Branches: refs/heads/trunk cbfe26370 -> b02a4b406 HADOOP-10758. KMS: add ACLs on per key basis. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b02a4b40 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b02a4b40 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b02a4b40 Branch: refs/heads/trunk Commit: b02a4b40610e93eef6559db09a11d287e859446d Parents: cbfe263 Author: Alejandro Abdelnur Authored: Wed Sep 10 14:26:15 2014 -0700 Committer: Alejandro Abdelnur Committed: Wed Sep 10 14:26:15 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../hadoop-kms/src/main/conf/kms-acls.xml | 38 +++ .../hadoop/crypto/key/kms/server/KMSACLs.java | 97 ++- .../crypto/key/kms/server/KMSConfiguration.java | 9 + .../hadoop/crypto/key/kms/server/KMSWebApp.java | 17 +- .../kms/server/KeyAuthorizationKeyProvider.java | 276 +++ .../hadoop-kms/src/site/apt/index.apt.vm| 106 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 236 +++- .../server/TestKeyAuthorizationKeyProvider.java | 218 +++ 9 files changed, 986 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b2157d6..3cea14a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -509,6 +509,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11057. checknative command to probe for winutils.exe on windows. (Xiaoyu Yao via cnauroth) +HADOOP-10758. KMS: add ACLs on per key basis. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml index cdff629..24a46b8 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml @@ -94,4 +94,42 @@ ACL for decrypt EncryptedKey CryptoExtension operations + + +default.key.acl.MANAGEMENT +* + + default ACL for MANAGEMENT operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.GENERATE_EEK +* + + default ACL for GENERATE_EEK operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.DECRYPT_EEK +* + + default ACL for DECRYPT_EEK operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.READ +* + + default ACL for READ operations for all key acls that are not + explicitly defined. + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b02a4b40/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java index 8a10bb2..530fe11 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java @@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; @@ -32,6 +34,7 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; /** * Provides access to the AccessControlLists used by KMS, @@ -39,7 +42,7 @@ import java.util.concurrent.TimeUnit; *
git commit: HADOOP-10758. KMS: add ACLs on per key basis. (tucu)
Repository: hadoop Updated Branches: refs/heads/branch-2 f58a076db -> 88e5549d9 HADOOP-10758. KMS: add ACLs on per key basis. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88e5549d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88e5549d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88e5549d Branch: refs/heads/branch-2 Commit: 88e5549d9017e1c919cc0d7199af8980b6aa6a24 Parents: f58a076 Author: Alejandro Abdelnur Authored: Wed Sep 10 14:26:15 2014 -0700 Committer: Alejandro Abdelnur Committed: Wed Sep 10 14:27:22 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../hadoop-kms/src/main/conf/kms-acls.xml | 38 +++ .../hadoop/crypto/key/kms/server/KMSACLs.java | 97 ++- .../crypto/key/kms/server/KMSConfiguration.java | 9 + .../hadoop/crypto/key/kms/server/KMSWebApp.java | 17 +- .../kms/server/KeyAuthorizationKeyProvider.java | 276 +++ .../hadoop-kms/src/site/apt/index.apt.vm| 106 +++ .../hadoop/crypto/key/kms/server/TestKMS.java | 236 +++- .../server/TestKeyAuthorizationKeyProvider.java | 218 +++ 9 files changed, 986 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e5549d/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index dc3f97d..53ab9e8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -173,6 +173,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11057. checknative command to probe for winutils.exe on windows. (Xiaoyu Yao via cnauroth) +HADOOP-10758. KMS: add ACLs on per key basis. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e5549d/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml index cdff629..24a46b8 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml @@ -94,4 +94,42 @@ ACL for decrypt EncryptedKey CryptoExtension operations + + +default.key.acl.MANAGEMENT +* + + default ACL for MANAGEMENT operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.GENERATE_EEK +* + + default ACL for GENERATE_EEK operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.DECRYPT_EEK +* + + default ACL for DECRYPT_EEK operations for all key acls that are not + explicitly defined. + + + + +default.key.acl.READ +* + + default ACL for READ operations for all key acls that are not + explicitly defined. + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e5549d/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java index 8a10bb2..530fe11 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java @@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyACLs; +import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; @@ -32,6 +34,7 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; /** * Provides access to the AccessControlLists used by KMS, @@ -39,7 +42,7 @@ import java.util.concurrent.TimeU
[3/3] git commit: HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as binary file but set it to the configuration as JSON file. (zxu via tucu)
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as
binary file but set it to the configuration as JSON file. (zxu via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e42b889b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e42b889b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e42b889b
Branch: refs/heads/branch-2
Commit: e42b889bdbccb691ebb942d56808c6624056884a
Parents: d0e2116
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:19:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:20:49 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/util/GenericOptionsParser.java| 2 +-
.../java/org/apache/hadoop/util/TestGenericOptionsParser.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e42b889b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b94198c..dc3f97d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -444,6 +444,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+HADOOP-9989. Bug introduced in HADOOP-9374, which parses the
-tokenCacheFile
+as binary file but set it to the configuration as JSON file. (zxu via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e42b889b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index 18acbf1..2a37dac 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -332,7 +332,7 @@ public class GenericOptionsParser {
}
UserGroupInformation.getCurrentUser().addCredentials(
Credentials.readTokenStorageFile(p, conf));
- conf.set("mapreduce.job.credentials.json", p.toString(),
+ conf.set("mapreduce.job.credentials.binary", p.toString(),
"from -tokenCacheFile command line option");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e42b889b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
index 779318a..2bc1915 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
@@ -249,7 +249,7 @@ public class TestGenericOptionsParser extends TestCase {
creds.writeTokenStorageFile(tmpPath, conf);
new GenericOptionsParser(conf, args);
-String fileName = conf.get("mapreduce.job.credentials.json");
+String fileName = conf.get("mapreduce.job.credentials.binary");
assertNotNull("files is null", fileName);
assertEquals("files option does not match", tmpPath.toString(), fileName);
[1/3] git commit: HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs doesn't work. (yzhangal via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7d9c45f77 -> e42b889bd
HDFS-6776. Using distcp to copy data between insecure and secure cluster via
webdhfs doesn't work. (yzhangal via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16a4558f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16a4558f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16a4558f
Branch: refs/heads/branch-2
Commit: 16a4558fda645c7960414f1e38457d9bb471d402
Parents: 7d9c45f
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:16:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:20:36 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../DelegationTokenSecretManager.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 3 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 -
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41
5 files changed, 65 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16a4558f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6171a39..2bd5cdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -481,6 +481,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
KeyProvider. (zhz via tucu)
+HDFS-6776. Using distcp to copy data between insecure and secure cluster
via webdhfs
+doesn't work. (yzhangal via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16a4558f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
index 175e3ed..8af7eba 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
@@ -402,8 +402,7 @@ public class DelegationTokenSecretManager
final Token token = namenode.getRpcServer(
).getDelegationToken(new Text(renewer));
if (token == null) {
- throw new IOException("Failed to get the token for " + renewer
- + ", user=" + ugi.getShortUserName());
+ return null;
}
final InetSocketAddress addr = namenode.getNameNodeAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16a4558f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 991885b..3949fbd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -283,6 +283,9 @@ public class NamenodeWebHdfsMethods {
final String renewer) throws IOException {
final Credentials c = DelegationTokenSecretManager.createCredentials(
namenode, ugi, renewer != null? renewer: ugi.getShortUserName());
+if (c == null) {
+ return null;
+}
final Token t =
c.getAllTokens().iterator().next();
Text kind = request.getScheme().equals("http") ?
WebHdfsFileSystem.TOKEN_KIND
: SWebHdfsFileSystem.TOKEN_KIND;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16a4558f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/We
[2/3] git commit: HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0e21165
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0e21165
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0e21165
Branch: refs/heads/branch-2
Commit: d0e211650244516abdef6ee212303af135167e39
Parents: 16a4558
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:18:03 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:20:43 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../authorize/DefaultImpersonationProvider.java | 2 +-
.../hadoop/security/authorize/TestProxyUsers.java| 15 +++
3 files changed, 18 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0e21165/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b414e53..b94198c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -442,6 +442,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10925. Compilation fails in native link0 function on Windows.
(cnauroth)
+HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0e21165/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
index ab1c390..b36ac80 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
@@ -123,7 +123,7 @@ public class DefaultImpersonationProvider implements
ImpersonationProvider {
MachineList MachineList = proxyHosts.get(
getProxySuperuserIpConfKey(realUser.getShortUserName()));
-if(!MachineList.includes(remoteAddress)) {
+if(MachineList == null || !MachineList.includes(remoteAddress)) {
throw new AuthorizationException("Unauthorized connection for
super-user: "
+ realUser.getUserName() + " from IP " + remoteAddress);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0e21165/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
index dbcac67..8ff4bfb 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
@@ -478,6 +478,21 @@ public class TestProxyUsers {
assertNotAuthorized(proxyUserUgi, "1.2.3.5");
}
+ @Test
+ public void testNoHostsForUsers() throws Exception {
+Configuration conf = new Configuration(false);
+conf.set("y." + REAL_USER_NAME + ".users",
+ StringUtils.join(",", Arrays.asList(AUTHORIZED_PROXY_USER_NAME)));
+ProxyUsers.refreshSuperUserGroupsConfiguration(conf, "y");
+
+UserGroupInformation realUserUgi = UserGroupInformation
+ .createRemoteUser(REAL_USER_NAME);
+UserGroupInformation proxyUserUgi =
UserGroupInformation.createProxyUserForTesting(
+ AUTHORIZED_PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
+
+// IP doesn't matter
+assertNotAuthorized(proxyUserUgi, "1.2.3.4");
+ }
private void assertNotAuthorized(UserGroupInformation proxyUgi, String host)
{
try {
[2/3] git commit: HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ee891aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ee891aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ee891aa
Branch: refs/heads/trunk
Commit: 9ee891aa90333bf18cba412400daa5834f15c41d
Parents: bbff44c
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:18:03 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:18:03 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../authorize/DefaultImpersonationProvider.java | 2 +-
.../hadoop/security/authorize/TestProxyUsers.java| 15 +++
3 files changed, 18 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c60a9b7..b015087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -777,6 +777,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10925. Compilation fails in native link0 function on Windows.
(cnauroth)
+HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
index ab1c390..b36ac80 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java
@@ -123,7 +123,7 @@ public class DefaultImpersonationProvider implements
ImpersonationProvider {
MachineList MachineList = proxyHosts.get(
getProxySuperuserIpConfKey(realUser.getShortUserName()));
-if(!MachineList.includes(remoteAddress)) {
+if(MachineList == null || !MachineList.includes(remoteAddress)) {
throw new AuthorizationException("Unauthorized connection for
super-user: "
+ realUser.getUserName() + " from IP " + remoteAddress);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee891aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
index dbcac67..8ff4bfb 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java
@@ -478,6 +478,21 @@ public class TestProxyUsers {
assertNotAuthorized(proxyUserUgi, "1.2.3.5");
}
+ @Test
+ public void testNoHostsForUsers() throws Exception {
+Configuration conf = new Configuration(false);
+conf.set("y." + REAL_USER_NAME + ".users",
+ StringUtils.join(",", Arrays.asList(AUTHORIZED_PROXY_USER_NAME)));
+ProxyUsers.refreshSuperUserGroupsConfiguration(conf, "y");
+
+UserGroupInformation realUserUgi = UserGroupInformation
+ .createRemoteUser(REAL_USER_NAME);
+UserGroupInformation proxyUserUgi =
UserGroupInformation.createProxyUserForTesting(
+ AUTHORIZED_PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
+
+// IP doesn't matter
+assertNotAuthorized(proxyUserUgi, "1.2.3.4");
+ }
private void assertNotAuthorized(UserGroupInformation proxyUgi, String host)
{
try {
[1/3] git commit: HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs doesn't work. (yzhangal via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 6dae4b430 -> b10094940
HDFS-6776. Using distcp to copy data between insecure and secure cluster via
webdhfs doesn't work. (yzhangal via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbff44cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbff44cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbff44cb
Branch: refs/heads/trunk
Commit: bbff44cb03d0150f990acc3b77170893241cc282
Parents: 6dae4b4
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:16:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:16:42 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../DelegationTokenSecretManager.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 3 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 -
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41
5 files changed, 65 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0b914ac..fa00d44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -739,6 +739,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
KeyProvider. (zhz via tucu)
+HDFS-6776. Using distcp to copy data between insecure and secure cluster
via webdhfs
+doesn't work. (yzhangal via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
index 175e3ed..8af7eba 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
@@ -402,8 +402,7 @@ public class DelegationTokenSecretManager
final Token token = namenode.getRpcServer(
).getDelegationToken(new Text(renewer));
if (token == null) {
- throw new IOException("Failed to get the token for " + renewer
- + ", user=" + ugi.getShortUserName());
+ return null;
}
final InetSocketAddress addr = namenode.getNameNodeAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 991885b..3949fbd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -283,6 +283,9 @@ public class NamenodeWebHdfsMethods {
final String renewer) throws IOException {
final Credentials c = DelegationTokenSecretManager.createCredentials(
namenode, ugi, renewer != null? renewer: ugi.getShortUserName());
+if (c == null) {
+ return null;
+}
final Token t =
c.getAllTokens().iterator().next();
Text kind = request.getScheme().equals("http") ?
WebHdfsFileSystem.TOKEN_KIND
: SWebHdfsFileSystem.TOKEN_KIND;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff44cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFi
[3/3] git commit: HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as binary file but set it to the configuration as JSON file. (zxu via tucu)
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as
binary file but set it to the configuration as JSON file. (zxu via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1009494
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1009494
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1009494
Branch: refs/heads/trunk
Commit: b100949404843ed245ef4e118291f55b3fdc81b8
Parents: 9ee891a
Author: Alejandro Abdelnur
Authored: Tue Sep 9 22:19:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Tue Sep 9 22:19:42 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/util/GenericOptionsParser.java| 2 +-
.../java/org/apache/hadoop/util/TestGenericOptionsParser.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b015087..b2157d6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -779,6 +779,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11077. NPE if hosts not specified in ProxyUsers. (gchanan via tucu)
+HADOOP-9989. Bug introduced in HADOOP-9374, which parses the
-tokenCacheFile
+as binary file but set it to the configuration as JSON file. (zxu via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index 18acbf1..2a37dac 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -332,7 +332,7 @@ public class GenericOptionsParser {
}
UserGroupInformation.getCurrentUser().addCredentials(
Credentials.readTokenStorageFile(p, conf));
- conf.set("mapreduce.job.credentials.json", p.toString(),
+ conf.set("mapreduce.job.credentials.binary", p.toString(),
"from -tokenCacheFile command line option");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1009494/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
index 779318a..2bc1915 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
@@ -249,7 +249,7 @@ public class TestGenericOptionsParser extends TestCase {
creds.writeTokenStorageFile(tmpPath, conf);
new GenericOptionsParser(conf, args);
-String fileName = conf.get("mapreduce.job.credentials.json");
+String fileName = conf.get("mapreduce.job.credentials.binary");
assertNotNull("files is null", fileName);
assertEquals("files option does not match", tmpPath.toString(), fileName);
git commit: HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on key rollover. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 876062ac2 -> d510cefd1
HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on
key rollover. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d510cefd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d510cefd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d510cefd
Branch: refs/heads/branch-2
Commit: d510cefd142ecdef124ff9efe85d4856a20c573a
Parents: 876062a
Author: Alejandro Abdelnur
Authored: Mon Sep 8 10:12:16 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 8 11:32:20 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../crypto/key/KeyProviderCryptoExtension.java | 11 ++
.../crypto/key/kms/KMSClientProvider.java | 9 +++-
.../hadoop/crypto/key/kms/ValueQueue.java | 13
.../hadoop/crypto/key/TestValueQueue.java | 14 +
...rKeyGeneratorKeyProviderCryptoExtension.java | 22
.../hadoop/crypto/key/kms/server/TestKMS.java | 17 +++
7 files changed, 88 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d510cefd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ed7b5f8..450053d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -430,6 +430,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
(Xiaoyu Yao via cnauroth)
+HADOOP-11071. KMSClientProvider should drain the local generated EEK cache
+on key rollover. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d510cefd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index e9d7caa..5d3281c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -179,6 +179,13 @@ public class KeyProviderCryptoExtension extends
throws IOException;
/**
+ * Drains the Queue for the provided key.
+ *
+ * @param keyName the key to drain the Queue for
+ */
+public void drain(String keyName);
+
+/**
* Generates a key material and encrypts it using the given key version
name
* and initialization vector. The generated key material is of the same
* length as the KeyVersion material of the latest key version
@@ -313,6 +320,10 @@ public class KeyProviderCryptoExtension extends
// NO-OP since the default version does not cache any keys
}
+@Override
+public void drain(String keyName) {
+ // NO-OP since the default version does not cache any keys
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d510cefd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 14593ed..ea191fc 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -590,7 +590,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
Map response = call(conn, jsonMaterial,
HttpURLConnection.HTTP_OK, Map.class);
-return parseJSONKeyVersion(response);
+KeyVersion keyVersion = parseJSONKeyVersion(response);
+encKeyVersionQueue.drain(name);
+return keyVersion;
}
@@ -713,6 +715,11 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
}
@Override
+ public void drain(String keyName) {
+encKeyVersionQueue.drain(keyN
git commit: HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on key rollover. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c1f832323 -> df8c84cba
HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on
key rollover. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df8c84cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df8c84cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df8c84cb
Branch: refs/heads/trunk
Commit: df8c84cba8512058f5097c6faeedf4b65cab3806
Parents: c1f8323
Author: Alejandro Abdelnur
Authored: Mon Sep 8 10:12:16 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 8 11:31:30 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../crypto/key/KeyProviderCryptoExtension.java | 11 ++
.../crypto/key/kms/KMSClientProvider.java | 9 +++-
.../hadoop/crypto/key/kms/ValueQueue.java | 13
.../hadoop/crypto/key/TestValueQueue.java | 14 +
...rKeyGeneratorKeyProviderCryptoExtension.java | 22
.../hadoop/crypto/key/kms/server/TestKMS.java | 17 +++
7 files changed, 88 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index fe011fd..0417b0a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -771,6 +771,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11073. Credential Provider related Unit Tests Failure on Windows.
(Xiaoyu Yao via cnauroth)
+HADOOP-11071. KMSClientProvider should drain the local generated EEK cache
+on key rollover. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index e2fb5cb..fed7e9e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -179,6 +179,13 @@ public class KeyProviderCryptoExtension extends
throws IOException;
/**
+ * Drains the Queue for the provided key.
+ *
+ * @param keyName the key to drain the Queue for
+ */
+public void drain(String keyName);
+
+/**
* Generates a key material and encrypts it using the given key version
name
* and initialization vector. The generated key material is of the same
* length as the KeyVersion material of the latest key version
@@ -313,6 +320,10 @@ public class KeyProviderCryptoExtension extends
// NO-OP since the default version does not cache any keys
}
+@Override
+public void drain(String keyName) {
+ // NO-OP since the default version does not cache any keys
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index acbe096..899b6c4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -590,7 +590,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
Map response = call(conn, jsonMaterial,
HttpURLConnection.HTTP_OK, Map.class);
-return parseJSONKeyVersion(response);
+KeyVersion keyVersion = parseJSONKeyVersion(response);
+encKeyVersionQueue.drain(name);
+return keyVersion;
}
@@ -713,6 +715,11 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
}
@Override
+ public void drain(String keyName) {
+encKeyVersionQueue.drain(keyN
git commit: HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8bf2a0de6 -> 035112f25
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035112f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035112f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035112f2
Branch: refs/heads/branch-2
Commit: 035112f25133343a55f9c65e0577a2230954dae8
Parents: 8bf2a0d
Author: Alejandro Abdelnur
Authored: Fri Sep 5 22:33:48 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:33:58 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++
.../hadoop/hdfs/DistributedFileSystem.java | 24 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43
4 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035112f2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0965b2c..5b74293 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -450,6 +450,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should
shutdown cluster (vinayakumarb)
+ HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
+ KeyProvider. (zhz via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035112f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 8daf912..e4215f0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
DFSHedgedReadMetrics getHedgedReadMetrics() {
return HEDGED_READ_METRIC;
}
+
+ public KeyProviderCryptoExtension getKeyProvider() {
+return provider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035112f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 6c04f01..bb671ce 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -1994,6 +1996,28 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public Token[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+Token[] tokens = super.addDelegationTokens(renewer, credentials);
+if (dfs.getKeyProvider() != null) {
+ KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
+ Token[] kpTokens = keyProviderDelegationTokenExtension.
+ addDelegationTokens(renewer, credentials);
+ if (tokens != null && kpTokens != null) {
+Token[] all = new Token[tokens.length + kpTokens.length];
+System.arraycopy(tokens, 0, all, 0, tokens.length);
+System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
+tokens = all;
+ } else {
+tokens = (tokens != null) ? tokens
git commit: HDFS-6986. DistributedFileSystem must get delegation tokens from configured KeyProvider. (zhz via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0f3c19c1b -> 3b35f8160
HDFS-6986. DistributedFileSystem must get delegation tokens from configured
KeyProvider. (zhz via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b35f816
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b35f816
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b35f816
Branch: refs/heads/trunk
Commit: 3b35f81603bbfae119762b50bcb46de70a421368
Parents: 0f3c19c
Author: Alejandro Abdelnur
Authored: Fri Sep 5 22:33:48 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:33:48 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++
.../hadoop/hdfs/DistributedFileSystem.java | 24 +++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 43
4 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0772ea6..333bdce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -711,6 +711,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6714. TestBlocksScheduledCounter#testBlocksScheduledCounter should
shutdown cluster (vinayakumarb)
+ HDFS-6986. DistributedFileSystem must get delegation tokens from
configured
+ KeyProvider. (zhz via tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 8daf912..e4215f0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3084,4 +3084,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
DFSHedgedReadMetrics getHedgedReadMetrics() {
return HEDGED_READ_METRIC;
}
+
+ public KeyProviderCryptoExtension getKeyProvider() {
+return provider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b35f816/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index bf7d62e..dbdf5c1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -84,8 +84,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -1946,6 +1948,28 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public Token[] addDelegationTokens(
+ final String renewer, Credentials credentials) throws IOException {
+Token[] tokens = super.addDelegationTokens(renewer, credentials);
+if (dfs.getKeyProvider() != null) {
+ KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension =
+ KeyProviderDelegationTokenExtension.
+ createKeyProviderDelegationTokenExtension(dfs.getKeyProvider());
+ Token[] kpTokens = keyProviderDelegationTokenExtension.
+ addDelegationTokens(renewer, credentials);
+ if (tokens != null && kpTokens != null) {
+Token[] all = new Token[tokens.length + kpTokens.length];
+System.arraycopy(tokens, 0, all, 0, tokens.length);
+System.arraycopy(kpTokens, 0, all, tokens.length, kpTokens.length);
+tokens = all;
+ } else {
+tokens = (tokens != null) ? tokens
[1/2] git commit: HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk e6420fec0 -> 0f3c19c1b
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f3c19c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f3c19c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f3c19c1
Branch: refs/heads/trunk
Commit: 0f3c19c1bb9e341d8aed132ba3eb9e7fc7588306
Parents: 71c8d73
Author: Alejandro Abdelnur
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java| 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9aef131..c77fddc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a4e336c..acbe096 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3c19c1/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
-UserGroupInformation clientUgi = UserGroupInformation.
+final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client",
keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
-UserGroupInformation.createRemoteUser("foo");
+UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction() {
@Override
publ
[2/2] git commit: HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf2a0de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf2a0de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf2a0de
Branch: refs/heads/branch-2
Commit: 8bf2a0de69547ac50b6e8c36ff7f13b028525641
Parents: e98c244
Author: Alejandro Abdelnur
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:01:13 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java| 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 492d41a..c799e20 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -424,6 +424,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index d459ba8..14593ed 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
-UserGroupInformation clientUgi = UserGroupInformation.
+final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client",
keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
-UserGroupInformation.createRemoteUser("foo");
+UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws Exception {
@@ -1179,7 +1179,7 @@ public class TestKMS {
[2/2] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c8d735
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c8d735
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c8d735
Branch: refs/heads/trunk
Commit: 71c8d735f5038e3b516947f12180d7568b6979dc
Parents: e6420fe
Author: Alejandro Abdelnur
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 21:59:12 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +---
4 files changed, 211 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 88804cd..9aef131 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -507,6 +507,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 3bb97c5..629ffda 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
-
+
-
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c8d735/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 000..5a6d4c5
--- /dev/null
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+server.getConnectors()[0].setHost(host);
+server.getConnectors()[0].setPort(port);
+ } else {
+SslSo
[1/2] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c47d72d8d -> 8bf2a0de6
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e98c2447
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e98c2447
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e98c2447
Branch: refs/heads/branch-2
Commit: e98c244730337477c0fe7c19c984ee4581ff567f
Parents: c47d72d
Author: Alejandro Abdelnur
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Sep 5 22:01:06 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml| 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +---
4 files changed, 211 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 724cfac..492d41a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -168,6 +168,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index 527454b..481f80e 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
-
+
-
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 000..5a6d4c5
--- /dev/null
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+server.getConnectors()[0].setHo
git commit: HDFS-6905. fs-encryption merge triggered release audit failures. (clamb via tucu)
Repository: hadoop Updated Branches: refs/heads/branch-2 7e7603927 -> 5dc45d529 HDFS-6905. fs-encryption merge triggered release audit failures. (clamb via tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5dc45d52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5dc45d52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5dc45d52 Branch: refs/heads/branch-2 Commit: 5dc45d529bb20f67b95f2876d103d12731be8df5 Parents: 7e76039 Author: Alejandro Abdelnur Authored: Fri Aug 22 05:17:22 2014 + Committer: Alejandro Abdelnur Committed: Thu Sep 4 11:06:58 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 2 ++ .../hadoop/hdfs/protocol/EncryptionZoneWithId.java | 17 + .../server/namenode/EncryptionFaultInjector.java | 17 + .../server/namenode/EncryptionZoneManager.java | 17 + 4 files changed, 53 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc45d52/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 25b6dc3..82bcf7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -425,6 +425,8 @@ Release 2.6.0 - UNRELEASED HDFS-2975. Rename with overwrite flag true can make NameNode to stuck in safemode on NN (crash + restart). (Yi Liu via umamahesh) + HDFS-6905. fs-encryption merge triggered release audit failures. (clamb via tucu) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc45d52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java index 7ed4884..e7fd2ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hdfs.protocol; import org.apache.commons.lang.builder.HashCodeBuilder; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc45d52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java index 2e65a89..27d8f50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,
git commit: Fixing HDFS CHANGES.txt, missing HDFS-6905 entry
Repository: hadoop Updated Branches: refs/heads/trunk 91d45f0f0 -> 1a0953614 Fixing HDFS CHANGES.txt, missing HDFS-6905 entry Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a095361 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a095361 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a095361 Branch: refs/heads/trunk Commit: 1a095361414ba660c139f33ae1eee430a3c3446c Parents: 91d45f0 Author: Alejandro Abdelnur Authored: Thu Sep 4 11:05:20 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Sep 4 11:07:08 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a095361/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8498b00..27b97cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -689,6 +689,8 @@ Release 2.6.0 - UNRELEASED HDFS-2975. Rename with overwrite flag true can make NameNode to stuck in safemode on NN (crash + restart). (Yi Liu via umamahesh) + HDFS-6905. fs-encryption merge triggered release audit failures. (clamb via tucu) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES
git commit: HADOOP-11060. Create a CryptoCodec test that verifies interoperability between the JCE and OpenSSL implementations. (hitliuyi via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 70b218748 -> b69a48c98
HADOOP-11060. Create a CryptoCodec test that verifies interoperability between
the JCE and OpenSSL implementations. (hitliuyi via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b69a48c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b69a48c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b69a48c9
Branch: refs/heads/trunk
Commit: b69a48c988c147abf192e36c99e2d4aecc116339
Parents: 70b2187
Author: Alejandro Abdelnur
Authored: Thu Sep 4 09:22:00 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 4 09:22:00 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/crypto/TestCryptoCodec.java | 69 +++-
.../apache/hadoop/crypto/TestCryptoStreams.java | 2 +-
3 files changed, 55 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69a48c9/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9645cba..f610c5d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -501,6 +501,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11015. Http server/client utils to propagate and recreate
Exceptions from server to client. (tucu)
+HADOOP-11060. Create a CryptoCodec test that verifies interoperability
+between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69a48c9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 49b5056..298f4ef 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -52,35 +52,40 @@ public class TestCryptoCodec {
private Configuration conf = new Configuration();
private int count = 1;
private int seed = new Random().nextInt();
+ private final String jceCodecClass =
+ "org.apache.hadoop.crypto.JceAesCtrCryptoCodec";
+ private final String opensslCodecClass =
+ "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec";
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-cryptoCodecTest(conf, seed, 0,
-"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
-cryptoCodecTest(conf, seed, count,
-"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
+Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
+cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
+cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
+cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass);
}
- @Test(timeout=120)
+ @Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
-cryptoCodecTest(conf, seed, 0,
-"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
-cryptoCodecTest(conf, seed, count,
-"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass);
+cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass);
+cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass);
}
private void cryptoCodecTest(Configuration conf, int seed, int count,
- String codecClass) throws IOException, GeneralSecurityException {
-CryptoCodec codec = null;
+ String encCodecClass, String decCodecClass) throws IOException,
+ GeneralSecurityException {
+CryptoCodec encCodec = null;
try {
- codec = (CryptoCodec)ReflectionUtils.newInstance(
- conf.getClassByName(codecClass), conf);
+ encCodec = (CryptoCodec)ReflectionUtils.newInstance(
+ conf.getClassByName(encCodecClass), conf);
} catch (ClassNotFoundExcep
git commit: HADOOP-11060. Create a CryptoCodec test that verifies interoperability between the JCE and OpenSSL implementations. (hitliuyi via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 dc2e38780 -> 2267ba1af
HADOOP-11060. Create a CryptoCodec test that verifies interoperability between
the JCE and OpenSSL implementations. (hitliuyi via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2267ba1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2267ba1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2267ba1a
Branch: refs/heads/branch-2
Commit: 2267ba1af72afdf846d4ee1a1cb7835838f79c41
Parents: dc2e387
Author: Alejandro Abdelnur
Authored: Thu Sep 4 09:22:00 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 4 09:22:10 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/crypto/TestCryptoCodec.java | 69 +++-
.../apache/hadoop/crypto/TestCryptoStreams.java | 2 +-
3 files changed, 55 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2267ba1a/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3cd0cf5..88095a5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -165,6 +165,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11015. Http server/client utils to propagate and recreate
Exceptions from server to client. (tucu)
+HADOOP-11060. Create a CryptoCodec test that verifies interoperability
+between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2267ba1a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 49b5056..298f4ef 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -52,35 +52,40 @@ public class TestCryptoCodec {
private Configuration conf = new Configuration();
private int count = 1;
private int seed = new Random().nextInt();
+ private final String jceCodecClass =
+ "org.apache.hadoop.crypto.JceAesCtrCryptoCodec";
+ private final String opensslCodecClass =
+ "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec";
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-cryptoCodecTest(conf, seed, 0,
-"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
-cryptoCodecTest(conf, seed, count,
-"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
+Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
+cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
+cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
+cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass);
}
- @Test(timeout=120)
+ @Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
-cryptoCodecTest(conf, seed, 0,
-"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
-cryptoCodecTest(conf, seed, count,
-"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass);
+cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass);
+cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass);
}
private void cryptoCodecTest(Configuration conf, int seed, int count,
- String codecClass) throws IOException, GeneralSecurityException {
-CryptoCodec codec = null;
+ String encCodecClass, String decCodecClass) throws IOException,
+ GeneralSecurityException {
+CryptoCodec encCodec = null;
try {
- codec = (CryptoCodec)ReflectionUtils.newInstance(
- conf.getClassByName(codecClass), conf);
+ encCodec = (CryptoCodec)ReflectionUtils.newInstance(
+ conf.getClassByName(encCodecClass), conf);
} catch (ClassNotFoundExcep
[1/2] git commit: HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b68818c4f -> dc2e38780
HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd55461c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd55461c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd55461c
Branch: refs/heads/branch-2
Commit: dd55461cdaa318966cf8df25820b62140221c44c
Parents: b68818c
Author: Alejandro Abdelnur
Authored: Thu Sep 4 09:08:31 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 4 09:14:02 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop/crypto/key/KeyProviderFactory.java | 36 ++--
.../crypto/key/TestKeyProviderFactory.java | 13 +++
3 files changed, 41 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd55461c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f26b6e2..b67e04d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -160,6 +160,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10863. KMS should have a blacklist for decrypting EEKs.
(asuresh via tucu)
+HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd55461c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index 799147e..cb63dcd 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -63,16 +63,10 @@ public abstract class KeyProviderFactory {
for(String path: conf.getStringCollection(KEY_PROVIDER_PATH)) {
try {
URI uri = new URI(path);
-boolean found = false;
-for(KeyProviderFactory factory: serviceLoader) {
- KeyProvider kp = factory.createProvider(uri, conf);
- if (kp != null) {
-result.add(kp);
-found = true;
-break;
- }
-}
-if (!found) {
+KeyProvider kp = get(uri, conf);
+if (kp != null) {
+ result.add(kp);
+} else {
throw new IOException("No KeyProviderFactory for " + uri + " in " +
KEY_PROVIDER_PATH);
}
@@ -83,4 +77,26 @@ public abstract class KeyProviderFactory {
}
return result;
}
+
+ /**
+ * Create a KeyProvider based on a provided URI.
+ *
+ * @param uri key provider URI
+ * @param conf configuration to initialize the key provider
+ * @return the key provider for the specified URI, or NULL if
+ * a provider for the specified URI scheme could not be found.
+ * @throws IOException thrown if the provider failed to initialize.
+ */
+ public static KeyProvider get(URI uri, Configuration conf)
+ throws IOException {
+KeyProvider kp = null;
+for (KeyProviderFactory factory : serviceLoader) {
+ kp = factory.createProvider(uri, conf);
+ if (kp != null) {
+break;
+ }
+}
+return kp;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd55461c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
index d72ac51..8c4c7b3 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
@@ -357,4 +357,17 @@ public class TestKeyProviderFactory {
}
}
+ @Test
+ public void testGetProviderViaURI() throws Exception {
+Configuration conf = new Configuration(false);
+URI uri = new URI(JavaKeyStoreProvider.SCHEME_NA
[2/2] git commit: HADOOP-11015. Http server/client utils to propagate and recreate Exceptions from server to client. (tucu)
HADOOP-11015. Http server/client utils to propagate and recreate Exceptions from server to client. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc2e3878 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc2e3878 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc2e3878 Branch: refs/heads/branch-2 Commit: dc2e38780b36063055eacae38e8094c126008d82 Parents: dd55461 Author: Alejandro Abdelnur Authored: Thu Sep 4 09:11:10 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Sep 4 09:14:07 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../dev-support/findbugsExcludeFile.xml | 2 +- .../crypto/key/kms/KMSClientProvider.java | 57 +- .../DelegationTokenAuthenticationFilter.java| 15 +- .../DelegationTokenAuthenticationHandler.java | 6 +- .../web/DelegationTokenAuthenticator.java | 20 +- .../apache/hadoop/util/HttpExceptionUtils.java | 185 +++ ...tionTokenAuthenticationHandlerWithMocks.java | 35 ++-- .../hadoop/util/TestHttpExceptionUtils.java | 167 + .../key/kms/server/KMSExceptionsProvider.java | 12 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 70 --- .../hadoop/fs/http/client/HttpFSUtils.java | 50 - .../hadoop/lib/wsrs/ExceptionProvider.java | 14 +- .../fs/http/client/BaseTestHttpFSWith.java | 4 +- .../fs/http/server/TestHttpFSServerNoACLs.java | 10 +- 15 files changed, 423 insertions(+), 227 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2e3878/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b67e04d..3cd0cf5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -162,6 +162,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu) +HADOOP-11015. Http server/client utils to propagate and recreate +Exceptions from server to client. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2e3878/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml -- diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index eead035..0181463 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -385,7 +385,7 @@ - + http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2e3878/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index c43dd86..d459ba8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -34,6 +34,7 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.http.client.utils.URIBuilder; import org.codehaus.jackson.map.ObjectMapper; @@ -44,7 +45,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import java.lang.reflect.Constructor; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URI; @@ -54,7 +54,6 @@ import java.net.URLEncoder; import java.security.GeneralSecurityException; import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; -import java.text.MessageFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -413,58 +412,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, return conn; } - // trick, riding on generics
[2/2] git commit: HADOOP-11015. Http server/client utils to propagate and recreate Exceptions from server to client. (tucu)
HADOOP-11015. Http server/client utils to propagate and recreate Exceptions from server to client. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70b21874 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70b21874 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70b21874 Branch: refs/heads/trunk Commit: 70b218748badf079c859c3af2b468a0b7b49c333 Parents: 41f1662 Author: Alejandro Abdelnur Authored: Thu Sep 4 09:11:10 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Sep 4 09:11:10 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../dev-support/findbugsExcludeFile.xml | 2 +- .../crypto/key/kms/KMSClientProvider.java | 57 +- .../DelegationTokenAuthenticationFilter.java| 15 +- .../DelegationTokenAuthenticationHandler.java | 6 +- .../web/DelegationTokenAuthenticator.java | 20 +- .../apache/hadoop/util/HttpExceptionUtils.java | 185 +++ ...tionTokenAuthenticationHandlerWithMocks.java | 35 ++-- .../hadoop/util/TestHttpExceptionUtils.java | 167 + .../key/kms/server/KMSExceptionsProvider.java | 12 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 70 --- .../hadoop/fs/http/client/HttpFSUtils.java | 50 - .../hadoop/lib/wsrs/ExceptionProvider.java | 14 +- .../fs/http/client/BaseTestHttpFSWith.java | 4 +- .../fs/http/server/TestHttpFSServerNoACLs.java | 10 +- 15 files changed, 423 insertions(+), 227 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/70b21874/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2e04917..9645cba 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -498,6 +498,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu) +HADOOP-11015. Http server/client utils to propagate and recreate +Exceptions from server to client. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/70b21874/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml -- diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index 1469034..204e6ab 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -367,7 +367,7 @@ - + http://git-wip-us.apache.org/repos/asf/hadoop/blob/70b21874/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index dc9e6cb..a4e336c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -34,6 +34,7 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.http.client.utils.URIBuilder; import org.codehaus.jackson.map.ObjectMapper; @@ -44,7 +45,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import java.lang.reflect.Constructor; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URI; @@ -54,7 +54,6 @@ import java.net.URLEncoder; import java.security.GeneralSecurityException; import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; -import java.text.MessageFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -413,58 +412,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, return conn; } - // trick, riding on generics to
[1/2] git commit: HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8f1a66857 -> 70b218748
HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41f1662d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41f1662d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41f1662d
Branch: refs/heads/trunk
Commit: 41f1662d467ec0b295b742bb80c87482504fbf25
Parents: 8f1a668
Author: Alejandro Abdelnur
Authored: Thu Sep 4 09:08:31 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Sep 4 09:09:39 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop/crypto/key/KeyProviderFactory.java | 36 ++--
.../crypto/key/TestKeyProviderFactory.java | 13 +++
3 files changed, 41 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41f1662d/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index e8d0f52..2e04917 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -496,6 +496,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10863. KMS should have a blacklist for decrypting EEKs.
(asuresh via tucu)
+HADOOP-11054. Add a KeyProvider instantiation based on a URI. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41f1662d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index 9855bc8..6ca0425 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -63,16 +63,10 @@ public abstract class KeyProviderFactory {
for(String path: conf.getStringCollection(KEY_PROVIDER_PATH)) {
try {
URI uri = new URI(path);
-boolean found = false;
-for(KeyProviderFactory factory: serviceLoader) {
- KeyProvider kp = factory.createProvider(uri, conf);
- if (kp != null) {
-result.add(kp);
-found = true;
-break;
- }
-}
-if (!found) {
+KeyProvider kp = get(uri, conf);
+if (kp != null) {
+ result.add(kp);
+} else {
throw new IOException("No KeyProviderFactory for " + uri + " in " +
KEY_PROVIDER_PATH);
}
@@ -83,4 +77,26 @@ public abstract class KeyProviderFactory {
}
return result;
}
+
+ /**
+ * Create a KeyProvider based on a provided URI.
+ *
+ * @param uri key provider URI
+ * @param conf configuration to initialize the key provider
+ * @return the key provider for the specified URI, or NULL if
+ * a provider for the specified URI scheme could not be found.
+ * @throws IOException thrown if the provider failed to initialize.
+ */
+ public static KeyProvider get(URI uri, Configuration conf)
+ throws IOException {
+KeyProvider kp = null;
+for (KeyProviderFactory factory : serviceLoader) {
+ kp = factory.createProvider(uri, conf);
+ if (kp != null) {
+break;
+ }
+}
+return kp;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41f1662d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
index d72ac51..8c4c7b3 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
@@ -357,4 +357,17 @@ public class TestKeyProviderFactory {
}
}
+ @Test
+ public void testGetProviderViaURI() throws Exception {
+Configuration conf = new Configuration(false);
+URI uri = new URI(JavaKeyStoreProvider.SCHEME_NA
git commit: HADOOP-10863. KMS should have a blacklist for decrypting EEKs. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 96a13c6d0 -> a7d8ede30
HADOOP-10863. KMS should have a blacklist for decrypting EEKs. (asuresh via
tucu)
Conflicts:
hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7d8ede3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7d8ede3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7d8ede3
Branch: refs/heads/branch-2
Commit: a7d8ede3091144cb16f84421b549c4619b3383aa
Parents: 96a13c6
Author: Alejandro Abdelnur
Authored: Wed Sep 3 15:08:55 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 3 15:20:28 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../security/authorize/AccessControlList.java | 12 ++-
.../hadoop/crypto/key/kms/server/KMS.java | 29 ++
.../hadoop/crypto/key/kms/server/KMSACLs.java | 55 +-
.../hadoop-kms/src/site/apt/index.apt.vm| 88 +++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 100 +--
.../crypto/key/kms/server/TestKMSACLs.java | 2 +-
7 files changed, 253 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d8ede3/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b2efecf..d803116 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -157,6 +157,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10990. Add missed NFSv3 request and response classes (brandonli)
+HADOOP-10863. KMS should have a blacklist for decrypting EEKs.
+(asuresh via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d8ede3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
index f78602a..d250df1 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
@@ -221,7 +221,13 @@ public class AccessControlList implements Writable {
return groups;
}
- public boolean isUserAllowed(UserGroupInformation ugi) {
+ /**
+ * Checks if a user represented by the provided {@link UserGroupInformation}
+ * is a member of the Access Control List
+ * @param ugi UserGroupInformation to check if contained in the ACL
+ * @return true if ugi is member of the list
+ */
+ public final boolean isUserInList(UserGroupInformation ugi) {
if (allAllowed || users.contains(ugi.getShortUserName())) {
return true;
} else {
@@ -234,6 +240,10 @@ public class AccessControlList implements Writable {
return false;
}
+ public boolean isUserAllowed(UserGroupInformation ugi) {
+return isUserInList(ugi);
+ }
+
/**
* Returns descriptive way of users and groups that are part of this ACL.
* Use {@link #getAclString()} to get the exact String that can be given to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d8ede3/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
index 608751a..43b07fe 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -26,10 +26,10 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
imp
git commit: HADOOP-10863. KMS should have a blacklist for decrypting EEKs. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1dcaba9a7 -> d9a03e272
HADOOP-10863. KMS should have a blacklist for decrypting EEKs. (asuresh via
tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a03e27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a03e27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a03e27
Branch: refs/heads/trunk
Commit: d9a03e272adbf3e9fde501610400f18fb4f6b865
Parents: 1dcaba9
Author: Alejandro Abdelnur
Authored: Wed Sep 3 15:08:55 2014 -0700
Committer: Alejandro Abdelnur
Committed: Wed Sep 3 15:08:55 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../security/authorize/AccessControlList.java | 12 ++-
.../hadoop/crypto/key/kms/server/KMS.java | 27 ++---
.../hadoop/crypto/key/kms/server/KMSACLs.java | 55 +-
.../hadoop-kms/src/site/apt/index.apt.vm| 88 +++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 100 +--
.../crypto/key/kms/server/TestKMSACLs.java | 2 +-
7 files changed, 252 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a03e27/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8e5f02a..0b9cfdc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -493,6 +493,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10990. Add missed NFSv3 request and response classes (brandonli)
+HADOOP-10863. KMS should have a blacklist for decrypting EEKs.
+(asuresh via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a03e27/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
index f78602a..d250df1 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
@@ -221,7 +221,13 @@ public class AccessControlList implements Writable {
return groups;
}
- public boolean isUserAllowed(UserGroupInformation ugi) {
+ /**
+ * Checks if a user represented by the provided {@link UserGroupInformation}
+ * is a member of the Access Control List
+ * @param ugi UserGroupInformation to check if contained in the ACL
+ * @return true if ugi is member of the list
+ */
+ public final boolean isUserInList(UserGroupInformation ugi) {
if (allAllowed || users.contains(ugi.getShortUserName())) {
return true;
} else {
@@ -234,6 +240,10 @@ public class AccessControlList implements Writable {
return false;
}
+ public boolean isUserAllowed(UserGroupInformation ugi) {
+return isUserInList(ugi);
+ }
+
/**
* Returns descriptive way of users and groups that are part of this ACL.
* Use {@link #getAclString()} to get the exact String that can be given to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a03e27/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
index faec70a..43b07fe 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -26,10 +26,10 @@ import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersi
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import
org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
+
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultVa
git commit: HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for generation/decryption of keys. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b61b78e5c -> 5889f4d5f
HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for
generation/decryption of keys. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5889f4d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5889f4d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5889f4d5
Branch: refs/heads/branch-2
Commit: 5889f4d5f33015ff0c57cc4fc319b2c113b36fe5
Parents: b61b78e
Author: Alejandro Abdelnur
Authored: Fri Aug 29 14:21:58 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 14:22:15 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 1 +
.../apache/hadoop/crypto/key/KeyProvider.java | 20
.../crypto/key/KeyProviderCryptoExtension.java | 51 +---
.../hadoop/crypto/key/KeyProviderExtension.java | 1 +
.../apache/hadoop/crypto/key/UserProvider.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 1 +
.../crypto/key/TestCachingKeyProvider.java | 6 +++
.../hadoop/crypto/key/TestKeyProvider.java | 17 ++-
...TestKeyProviderDelegationTokenExtension.java | 11 -
10 files changed, 93 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5889f4d5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index eba1dff..9df7dbb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -146,6 +146,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
6.x version. (rkanter via tucu)
+HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for
+generation/decryption of keys. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5889f4d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 2503151..30583eb 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -108,6 +108,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final Map cache = new HashMap();
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException
{
+super(conf);
this.uri = uri;
path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5889f4d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 9c46875..a8b9414 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -56,6 +56,8 @@ public abstract class KeyProvider {
"hadoop.security.key.default.bitlength";
public static final int DEFAULT_BITLENGTH = 128;
+ private final Configuration conf;
+
/**
* The combination of both the key version name and the key material.
*/
@@ -354,6 +356,24 @@ public abstract class KeyProvider {
}
/**
+ * Constructor.
+ *
+ * @param conf configuration for the provider
+ */
+ public KeyProvider(Configuration conf) {
+this.conf = new Configuration(conf);
+ }
+
+ /**
+ * Return the provider configuration.
+ *
+ * @return the provider configuration
+ */
+ public Configuration getConf() {
+return conf;
+ }
+
+ /**
* A helper function to create an options object.
* @param conf the configuration to use
* @return a new options object
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5889f4d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hado
git commit: HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for generation/decryption of keys. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk b03653f9a -> c60da4d3b
HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for
generation/decryption of keys. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c60da4d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c60da4d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c60da4d3
Branch: refs/heads/trunk
Commit: c60da4d3b31e5fa0c4b27cf75ab7ed4add56396a
Parents: b03653f
Author: Alejandro Abdelnur
Authored: Fri Aug 29 14:21:58 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 14:21:58 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 1 +
.../apache/hadoop/crypto/key/KeyProvider.java | 20
.../crypto/key/KeyProviderCryptoExtension.java | 51 +---
.../hadoop/crypto/key/KeyProviderExtension.java | 1 +
.../apache/hadoop/crypto/key/UserProvider.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 1 +
.../crypto/key/TestCachingKeyProvider.java | 6 +++
.../hadoop/crypto/key/TestKeyProvider.java | 17 ++-
...TestKeyProviderDelegationTokenExtension.java | 13 +++--
10 files changed, 94 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1930e5d..2bc3e4b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -476,6 +476,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
6.x version. (rkanter via tucu)
+HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for
+generation/decryption of keys. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 2503151..30583eb 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -108,6 +108,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final Map cache = new HashMap();
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException
{
+super(conf);
this.uri = uri;
path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a34ae10..36ccbad 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -56,6 +56,8 @@ public abstract class KeyProvider {
"hadoop.security.key.default.bitlength";
public static final int DEFAULT_BITLENGTH = 128;
+ private final Configuration conf;
+
/**
* The combination of both the key version name and the key material.
*/
@@ -354,6 +356,24 @@ public abstract class KeyProvider {
}
/**
+ * Constructor.
+ *
+ * @param conf configuration for the provider
+ */
+ public KeyProvider(Configuration conf) {
+this.conf = new Configuration(conf);
+ }
+
+ /**
+ * Return the provider configuration.
+ *
+ * @return the provider configuration
+ */
+ public Configuration getConf() {
+return conf;
+ }
+
+ /**
* A helper function to create an options object.
* @param conf the configuration to use
* @return a new options object
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hado
git commit: HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest 6.x version. (rkanter via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c686aa353 -> b1dce2aa2
HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest 6.x
version. (rkanter via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1dce2aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1dce2aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1dce2aa
Branch: refs/heads/trunk
Commit: b1dce2aa21d9692accdec710ef044d2a2e04ba33
Parents: c686aa3
Author: Alejandro Abdelnur
Authored: Fri Aug 29 11:51:23 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 11:53:22 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-kms/pom.xml| 1 -
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
hadoop-project/pom.xml | 2 ++
4 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1dce2aa/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6376364..1930e5d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -473,6 +473,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11005. Fix HTTP content type for ReconfigurationServlet.
(Lei Xu via wang)
+HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
+6.x version. (rkanter via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1dce2aa/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index b65e67a..b1ca307 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -34,7 +34,6 @@
Apache Hadoop KMS
-6.0.36
${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/kms/tomcat
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1dce2aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 8701bb0..24fa87b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -34,7 +34,6 @@
Apache Hadoop HttpFS
-6.0.36
REPO NOT AVAIL
REPO NOT AVAIL
REVISION NOT AVAIL
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1dce2aa/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e9adc31..5aa54a7 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -67,6 +67,8 @@
${env.HADOOP_PROTOC_PATH}
3.4.6
+
+6.0.41
git commit: HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest 6.x version. (rkanter via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 73a0e4665 -> 09a0ad328
HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest 6.x
version. (rkanter via tucu)
(cherry picked from commit 189abddf0b68ab43978dacaf3a9bf6ee7169cf78)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09a0ad32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09a0ad32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09a0ad32
Branch: refs/heads/branch-2
Commit: 09a0ad328f9adbb7b3c519ea4fbef27a0d97992f
Parents: 73a0e46
Author: Alejandro Abdelnur
Authored: Fri Aug 29 11:51:23 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 11:53:13 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-kms/pom.xml| 1 -
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
hadoop-project/pom.xml | 2 ++
4 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a0ad32/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5c52255..eba1dff 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -143,6 +143,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11005. Fix HTTP content type for ReconfigurationServlet.
(Lei Xu via wang)
+HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
+6.x version. (rkanter via tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a0ad32/hadoop-common-project/hadoop-kms/pom.xml
--
diff --git a/hadoop-common-project/hadoop-kms/pom.xml
b/hadoop-common-project/hadoop-kms/pom.xml
index edfd760..7d516ec 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -34,7 +34,6 @@
Apache Hadoop KMS
-6.0.36
${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/kms/tomcat
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a0ad32/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 2bf85bf..4d20fa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -34,7 +34,6 @@
Apache Hadoop HttpFS
-6.0.36
REPO NOT AVAIL
REPO NOT AVAIL
REVISION NOT AVAIL
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a0ad32/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2d60165..08959e3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -67,6 +67,8 @@
${env.HADOOP_PROTOC_PATH}
3.4.6
+
+6.0.41
git commit: HADOOP-10911. hadoop.auth cookie after HADOOP-10710 still not proper according to RFC2109. (gchanan via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 3de66011c -> 156e6a4f8
HADOOP-10911. hadoop.auth cookie after HADOOP-10710 still not proper according
to RFC2109. (gchanan via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/156e6a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/156e6a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/156e6a4f
Branch: refs/heads/trunk
Commit: 156e6a4f8aed69febec408af423b2a8ac313c643
Parents: 3de6601
Author: Alejandro Abdelnur
Authored: Fri Aug 29 11:06:51 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 11:23:23 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 10 ++
.../server/AuthenticationFilter.java| 4 +-
.../client/AuthenticatorTestCase.java | 137 ++-
.../client/TestKerberosAuthenticator.java | 58 +++-
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-project/pom.xml | 10 ++
6 files changed, 210 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156e6a4f/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index 2ff51d6f..564518c 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -62,6 +62,16 @@
jetty
test
+
+ org.apache.tomcat.embed
+ tomcat-embed-core
+ test
+
+
+ org.apache.tomcat.embed
+ tomcat-embed-logging-juli
+ test
+
javax.servlet
servlet-api
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156e6a4f/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 316cd60..9330444 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -519,9 +519,7 @@ public class AuthenticationFilter implements Filter {
StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE)
.append("=");
if (token != null && token.length() > 0) {
- sb.append("\"")
- .append(token)
- .append("\"");
+ sb.append(token);
}
sb.append("; Version=1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156e6a4f/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
index 4e4ecc4..8f35e13 100644
---
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
+++
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
@@ -13,7 +13,22 @@
*/
package org.apache.hadoop.security.authentication.client;
+import org.apache.catalina.deploy.FilterDef;
+import org.apache.catalina.deploy.FilterMap;
+import org.apache.catalina.startup.Tomcat;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.params.AuthPolicy;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
+import org.apache.http.util.EntityUtils;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.FilterHolder;
@@
git commit: HADOOP-10911. hadoop.auth cookie after HADOOP-10710 still not proper according to RFC2109. (gchanan via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aeb8667a0 -> 54202383a
HADOOP-10911. hadoop.auth cookie after HADOOP-10710 still not proper according
to RFC2109. (gchanan via tucu)
(cherry picked from commit 6040810df82669f140033d3c6366892640798671)
Conflicts:
hadoop-project/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54202383
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54202383
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54202383
Branch: refs/heads/branch-2
Commit: 54202383a9627415c822bddd2947a1a179b6319f
Parents: aeb8667
Author: Alejandro Abdelnur
Authored: Fri Aug 29 11:06:51 2014 -0700
Committer: Alejandro Abdelnur
Committed: Fri Aug 29 11:23:14 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 10 ++
.../server/AuthenticationFilter.java| 4 +-
.../client/AuthenticatorTestCase.java | 137 ++-
.../client/TestKerberosAuthenticator.java | 58 +++-
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-project/pom.xml | 15 ++
6 files changed, 215 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54202383/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index e7de14c..20304e1 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -67,6 +67,16 @@
jetty
test
+
+ org.apache.tomcat.embed
+ tomcat-embed-core
+ test
+
+
+ org.apache.tomcat.embed
+ tomcat-embed-logging-juli
+ test
+
javax.servlet
servlet-api
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54202383/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 316cd60..9330444 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -519,9 +519,7 @@ public class AuthenticationFilter implements Filter {
StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE)
.append("=");
if (token != null && token.length() > 0) {
- sb.append("\"")
- .append(token)
- .append("\"");
+ sb.append(token);
}
sb.append("; Version=1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54202383/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
index 4e4ecc4..8f35e13 100644
---
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
+++
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java
@@ -13,7 +13,22 @@
*/
package org.apache.hadoop.security.authentication.client;
+import org.apache.catalina.deploy.FilterDef;
+import org.apache.catalina.deploy.FilterMap;
+import org.apache.catalina.startup.Tomcat;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.params.AuthPolicy;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
+import org.apache.http.util.EntityUtils;
import org.mortbay.
git commit: Removing CHANGES-fs-encryption.txt files
ased on inodes (clamb) - -HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao) - -HDFS-6635. Refactor encryption zone functionality into new -EncryptionZoneManager class. (wang) - -HDFS-6474. Namenode needs to get the actual keys and iv from the -KeyProvider. (wang) - -HDFS-6619. Clean up encryption-related tests. (wang) - -HDFS-6405. Test Crypto streams in HDFS. (yliu via wang) - -HDFS-6490. Fix the keyid format for generated keys in -FSNamesystem.createEncryptionZone (clamb) - -HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode. -(wang) - -HDFS-6718. Remove EncryptionZoneManager lock. (wang) - -HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang) - -HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in -EZManager#createEncryptionZone. (clamb) - -HDFS-6724. Decrypt EDEK before creating -CryptoInputStream/CryptoOutputStream. (wang) - -HDFS-6509. Create a special /.reserved/raw directory for raw access to -encrypted data. (clamb via wang) - -HDFS-6771. Require specification of an encryption key when creating -an encryption zone. (wang) - -HDFS-6730. Create a .RAW extended attribute namespace. (clamb) - -HDFS-6692. Add more HDFS encryption tests. (wang) - -HDFS-6780. Batch the encryption zones listing API. (wang) - -HDFS-6394. HDFS encryption documentation. (wang) - -HDFS-6834. Improve the configuration guidance in DFSClient when there -are no Codec classes found in configs. (umamahesh) - -HDFS-6546. Add non-superuser capability to get the encryption zone -for a specific path. (clamb) - - OPTIMIZATIONS - - BUG FIXES - -HDFS-6733. Creating encryption zone results in NPE when -KeyProvider is null. (clamb) - -HDFS-6785. Should not be able to create encryption zone using path -to a non-directory file. (clamb) - -HDFS-6807. Fix TestReservedRawPaths. (clamb) - -HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured -as boolean. (umamahesh) - -HDFS-6817. Fix findbugs and other warnings. (yliu) - -HDFS-6839. Fix TestCLI to expect new output. (clamb) http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a65717f/hadoop-mapreduce-project/CHANGES-fs-encryption.txt -- diff --git a/hadoop-mapreduce-project/CHANGES-fs-encryption.txt b/hadoop-mapreduce-project/CHANGES-fs-encryption.txt deleted file mode 100644 index 3e1718e..000 --- a/hadoop-mapreduce-project/CHANGES-fs-encryption.txt +++ /dev/null @@ -1,20 +0,0 @@ -Hadoop MapReduce Change Log - -fs-encryption (Unreleased) - - INCOMPATIBLE CHANGES - - NEW FEATURES - -MAPREDUCE-5890. Support for encrypting Intermediate -data and spills in local filesystem. (asuresh via tucu) - - IMPROVEMENTS - -MAPREDUCE-6007. Add support to distcp to preserve raw.* namespace -extended attributes. (clamb) - -HDFS-6872. Fix TestOptionsParser. (clamb) - - BUG FIXES -
[2/2] git commit: Fix up CHANGES.txt for HDFS-6134, HADOOP-10150 and related JIRAs following merge to branch-2
CryptoCodec using JNI to OpenSSL. + (Yi Liu via cmccabe) + + HADOOP-10803. Update OpensslCipher#getInstance to accept CipherSuite#name + format. (Yi Liu) + + HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to + JCE if non native support. (Yi Liu) + + HADOOP-10870. Failed to load OpenSSL cipher error logs on systems with old + openssl versions (cmccabe) + + HADOOP-10853. Refactor get instance of CryptoCodec and support create via + algorithm/mode/padding. (Yi Liu) + + HADOOP-10919. Copy command should preserve raw.* namespace + extended attributes. (clamb) + + HDFS-6873. Constants in CommandWithDestination should be static. (clamb) + + HADOOP-10871. incorrect prototype in OpensslSecureRandom.c (cmccabe) + + HADOOP-10886. CryptoCodec#getCodecclasses throws NPE when configurations not + loaded. (umamahesh) +-- + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a7404c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1bb6025..2c56407 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -255,99 +255,6 @@ Trunk (Unreleased) HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. (Vinayakumar B via wheat 9) - BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS - -HDFS-6387. HDFS CLI admin tool for creating & deleting an -encryption zone. (clamb) - -HDFS-6386. HDFS Encryption Zones (clamb) - -HDFS-6388. HDFS integration with KeyProvider. (clamb) - -HDFS-6473. Protocol and API for Encryption Zones (clamb) - -HDFS-6392. Wire crypto streams for encrypted files in -DFSClient. (clamb and yliu) - -HDFS-6476. Print out the KeyProvider after finding KP successfully on -startup. (Juan Yu via wang) - -HDFS-6391. Get the Key/IV from the NameNode for encrypted files in -DFSClient. (Charles Lamb and wang) - -HDFS-6389. Rename restrictions for encryption zones. (clamb) - -HDFS-6605. Client server negotiation of cipher suite. (wang) - -HDFS-6625. Remove the Delete Encryption Zone function (clamb) - -HDFS-6516. List of Encryption Zones should be based on inodes (clamb) - -HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao) - -HDFS-6635. Refactor encryption zone functionality into new -EncryptionZoneManager class. (wang) - -HDFS-6474. Namenode needs to get the actual keys and iv from the -KeyProvider. (wang) - -HDFS-6619. Clean up encryption-related tests. (wang) - -HDFS-6405. Test Crypto streams in HDFS. (yliu via wang) - -HDFS-6490. Fix the keyid format for generated keys in -FSNamesystem.createEncryptionZone (clamb) - -HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode. -(wang) - -HDFS-6718. Remove EncryptionZoneManager lock. (wang) - -HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang) - -HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in -EZManager#createEncryptionZone. (clamb) - -HDFS-6724. Decrypt EDEK before creating -CryptoInputStream/CryptoOutputStream. (wang) - -HDFS-6509. Create a special /.reserved/raw directory for raw access to -encrypted data. (clamb via wang) - -HDFS-6771. Require specification of an encryption key when creating -an encryption zone. (wang) - -HDFS-6730. Create a .RAW extended attribute namespace. (clamb) - -HDFS-6692. Add more HDFS encryption tests. (wang) - -HDFS-6780. Batch the encryption zones listing API. (wang) - -HDFS-6394. HDFS encryption documentation. (wang) - -HDFS-6834. Improve the configuration guidance in DFSClient when there -are no Codec classes found in configs. (umamahesh) - -HDFS-6546. Add non-superuser capability to get the encryption zone -for a specific path. (clamb) - -HDFS-6733. Creating encryption zone results in NPE when -KeyProvider is null. (clamb) - -HDFS-6785. Should not be able to create encryption zone using path -to a non-directory file. (clamb) - -HDFS-6807. Fix TestReservedRawPaths. (clamb) - -HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured -as boolean. (umamahesh) - -HDFS-6817. Fix findbugs and other warnings. (yliu) - -HDFS-6839. Fix TestCLI to expect new output. (clamb) - -HDFS-6905. fs-encryption merge triggered release audit failures. (clamb via tucu) - HDFS-6694. TestPipelinesFailover.testPipelineRecoveryStress tests fail intermittently with various symptoms - debugging patch. (Yongjun Zhang via Arpit Agarwal) @@ -661,6 +568,98 @@ Release 2.6.0 - UNRELEASED HDFS-6902
[1/2] git commit: Fixing CHANGES.txt, moving HADOOP-8815 to 2.6.0 release
Repository: hadoop Updated Branches: refs/heads/trunk d1ae479aa -> d9a7404c3 Fixing CHANGES.txt, moving HADOOP-8815 to 2.6.0 release Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88c5e214 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88c5e214 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88c5e214 Branch: refs/heads/trunk Commit: 88c5e2141c4e85c2cac9463aaf68091a0e93302e Parents: d1ae479 Author: Alejandro Abdelnur Authored: Wed Aug 27 09:03:11 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Aug 28 15:07:57 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c5e214/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 641635b..2d794cf 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -232,9 +232,6 @@ Trunk (Unreleased) HADOOP-8813. Add InterfaceAudience and InterfaceStability annotations to RPC Server and Client classes. (Brandon Li via suresh) -HADOOP-8815. RandomDatum needs to override hashCode(). -(Brandon Li via suresh) - HADOOP-8436. NPE In getLocalPathForWrite ( path, conf ) when the required context item is not configured (Brahma Reddy Battula via harsh) @@ -704,6 +701,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10989. Work around buggy getgrouplist() implementations on Linux that return 0 on failure. (cnauroth) +HADOOP-8815. RandomDatum needs to override hashCode(). +(Brandon Li via suresh) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES
[07/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index b9af35e..c49d210 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
+.EncryptedKeyVersion;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
@@ -76,6 +81,7 @@ import java.net.Socket;
import java.net.SocketAddress;
import java.net.URI;
import java.net.UnknownHostException;
+import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -95,6 +101,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.CryptoInputStream;
+import org.apache.hadoop.crypto.CryptoOutputStream;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStorageLocation;
import org.apache.hadoop.fs.CacheFlag;
@@ -102,6 +113,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
@@ -140,6 +152,9 @@ import
org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
+import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -249,7 +264,11 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
private static final DFSHedgedReadMetrics HEDGED_READ_METRIC =
new DFSHedgedReadMetrics();
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
-
+ private final CryptoCodec codec;
+ @VisibleForTesting
+ List cipherSuites;
+ @VisibleForTesting
+ KeyProviderCryptoExtension provider;
/**
* DFSClient configuration
*/
@@ -581,7 +600,17 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
-
+this.codec = CryptoCodec.getInstance(conf);
+this.cipherSuites = Lists.newArrayListWithCapacity(1);
+if (codec != null) {
+ cipherSuites.add(codec.getCipherSuite());
+}
+provider = DFSUtil.createKeyProviderCryptoExtension(conf);
+if (provider == null) {
+ LOG.info("No KeyProvider found.");
+} else {
+ LOG.info("Found KeyProvider: " + provider.toString());
+}
int numResponseToDrop = conf.getInt(
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
@@ -1280,7 +1309,93 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
return volumeBlockLocations;
}
-
+
+ /**
+ * Decrypts a EDEK by consulting the KeyProvider.
+ */
+ private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
+ feInfo) throws IOException {
+if (provider == null) {
+ throw new IOException("No KeyP
[09/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
new file mode 100644
index 000..4ca79b3
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.GeneralSecurityException;
+import java.security.SecureRandom;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.crypto.random.OsSecureRandom;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Implement the AES-CTR crypto codec using JNI into OpenSSL.
+ */
+@InterfaceAudience.Private
+public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec {
+ private static final Log LOG =
+ LogFactory.getLog(OpensslAesCtrCryptoCodec.class.getName());
+
+ private Configuration conf;
+ private Random random;
+
+ public OpensslAesCtrCryptoCodec() {
+String loadingFailureReason = OpensslCipher.getLoadingFailureReason();
+if (loadingFailureReason != null) {
+ throw new RuntimeException(loadingFailureReason);
+}
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+this.conf = conf;
+final Class klass = conf.getClass(
+HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, OsSecureRandom.class,
+Random.class);
+try {
+ random = ReflectionUtils.newInstance(klass, conf);
+} catch (Exception e) {
+ LOG.info("Unable to use " + klass.getName() + ". Falling back to " +
+ "Java SecureRandom.", e);
+ this.random = new SecureRandom();
+}
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+try {
+ Closeable r = (Closeable) this.random;
+ r.close();
+} catch (ClassCastException e) {
+}
+super.finalize();
+ }
+
+ @Override
+ public Configuration getConf() {
+return conf;
+ }
+
+ @Override
+ public Encryptor createEncryptor() throws GeneralSecurityException {
+return new OpensslAesCtrCipher(OpensslCipher.ENCRYPT_MODE);
+ }
+
+ @Override
+ public Decryptor createDecryptor() throws GeneralSecurityException {
+return new OpensslAesCtrCipher(OpensslCipher.DECRYPT_MODE);
+ }
+
+ @Override
+ public void generateSecureRandom(byte[] bytes) {
+random.nextBytes(bytes);
+ }
+
+ private static class OpensslAesCtrCipher implements Encryptor, Decryptor {
+private final OpensslCipher cipher;
+private final int mode;
+private boolean contextReset = false;
+
+public OpensslAesCtrCipher(int mode) throws GeneralSecurityException {
+ this.mode = mode;
+ cipher = OpensslCipher.getInstance(SUITE.getName());
+}
+
+@Override
+public void init(byte[] key, byte[] iv) throws IOException {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(iv);
+ contextReset = false;
+ cipher.init(mode, key, iv);
+}
+
+/**
+ * AES-CTR will consume all of the input data. It requires enough space in
+ * the destination buffer to encrypt entire input buffer.
+ */
+@Override
+public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
+throws IOException {
+ process(inBuffer, outBuffer);
+}
+
+/**
+ * AES-CTR will consume all of the input data. It requires enough space in
+ * the destination buffer to decrypt entire input buffer.
+
[04/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
index af6132b..8f0611b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm
@@ -32,7 +32,7 @@ Extended Attributes in HDFS
** {Namespaces and Permissions}
- In HDFS, as in Linux, there are four valid namespaces: <<>>,
<<>>, <<>>, and <<>>. Each of these namespaces have
different access restrictions.
+ In HDFS, there are five valid namespaces: <<>>, <<>>,
<<>>, <<>>, and <<>>. Each of these namespaces have
different access restrictions.
The <<>> namespace is the namespace that will commonly be used by
client applications. Access to extended attributes in the user namespace is
controlled by the corresponding file permissions.
@@ -42,6 +42,8 @@ Extended Attributes in HDFS
The <<>> namespace is reserved for internal HDFS use. This
namespace is not accessible through userspace methods. It is currently unused.
+ The <<>> namespace is reserved for internal system attributes that
sometimes need to be exposed. Like <<>> namespace attributes they are
not visible to the user except when <<>>/<<>> is called on
a file or directory in the <<>> HDFS directory hierarchy. These
attributes can only be accessed by the superuser. An example of where <<>>
namespace extended attributes are used is the <<>> utility. Encryption
zone meta data is stored in <<>> extended attributes, so as long as the
administrator uses <<>> pathnames in source and target, the
encrypted files in the encryption zones are transparently copied.
+
* {Interacting with extended attributes}
The Hadoop shell has support for interacting with extended attributes via
<<>> and <<>>. These commands are
styled after the Linux
{{{http://www.bestbits.at/acl/man/man1/getfattr.txt}getfattr(1)}} and
{{{http://www.bestbits.at/acl/man/man1/setfattr.txt}setfattr(1)}} commands.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm
b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm
new file mode 100644
index 000..3689a77
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm
@@ -0,0 +1,206 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ Hadoop Distributed File System-${project.version} - Transparent Encryption
in HDFS
+ ---
+ ---
+ ${maven.build.timestamp}
+
+Transparent Encryption in HDFS
+
+%{toc|section=1|fromDepth=2|toDepth=3}
+
+* {Overview}
+
+ HDFS implements , encryption.
+ Once configured, data read from and written to HDFS is
encrypted and decrypted without requiring changes to user application code.
+ This encryption is also , which means the data can only be
encrypted and decrypted by the client.
+ HDFS never stores or has access to unencrypted data or data encryption keys.
+ This satisfies two typical requirements for encryption:
(meaning data on persistent media, such as a disk) as well as (e.g. when data is travelling over the network).
+
+* {Use Cases}
+
+ Data encryption is required by a number of different government, financial,
and regulatory entities.
+ For example, the health-care industry has HIPAA regulations, the card
payment industry has PCI DSS regulations, and the US government has FISMA
regulations.
+ Having transparent encryption built into HDFS makes it easier for
organizations to comply with these regulations.
+
+ Encryption can also be performed at the application-level, but by
integrating it into HDFS, existing applications can operate on encrypted data
without changes.
+ This integrated architecture implies stronger encrypted file semantics and
better coordination with other HDFS functions.
+
+* {Architecture}
+
+** {Key Management Server, KeyProvider, EDEKs}
+
+ A new cluster service is required to store, manage, and access encryption
keys: the Hadoop
[08/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
new file mode 100644
index 000..f5acc73
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
@@ -0,0 +1,721 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.EnumSet;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.ReadOption;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.Syncable;
+import org.apache.hadoop.io.ByteBufferPool;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.RandomDatum;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public abstract class CryptoStreamsTestBase {
+ protected static final Log LOG = LogFactory.getLog(
+ CryptoStreamsTestBase.class);
+
+ protected static CryptoCodec codec;
+ private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
+ private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+
+ protected static final int count = 1;
+ protected static int defaultBufferSize = 8192;
+ protected static int smallBufferSize = 1024;
+ private byte[] data;
+ private int dataLen;
+
+ @Before
+ public void setUp() throws IOException {
+// Generate data
+final int seed = new Random().nextInt();
+final DataOutputBuffer dataBuf = new DataOutputBuffer();
+final RandomDatum.Generator generator = new RandomDatum.Generator(seed);
+for(int i = 0; i < count; ++i) {
+ generator.next();
+ final RandomDatum key = generator.getKey();
+ final RandomDatum value = generator.getValue();
+
+ key.write(dataBuf);
+ value.write(dataBuf);
+}
+LOG.info("Generated " + count + " records");
+data = dataBuf.getData();
+dataLen = dataBuf.getLength();
+ }
+
+ protected void writeData(OutputStream out) throws Exception {
+out.write(data, 0, dataLen);
+out.close();
+ }
+
+ protected int getDataLen() {
+return dataLen;
+ }
+
+ private int readAll(InputStream in, byte[] b, int off, int len)
+ throws IOException {
+int n = 0;
+int total = 0;
+while (n != -1) {
+ total += n;
+ if (total >= len) {
+break;
+ }
+ n = in.read(b, off + total, len - total);
+}
+
+return total;
+ }
+
+ protected OutputStream getOutputStream(int bufferSize) throws IOException {
+return getOutputStream(bufferSize, key, iv);
+ }
+
+ protected abstract OutputStream getOutputStream(int bufferSize, byte[] key,
+ byte[] iv) throws IOException;
+
+ protected InputStream getInputStream(int bufferSize) throws IOException {
+return getInputStream(bufferSize, key, iv);
+ }
+
+ protected abstract InputStream getInputStream(int bufferSize, byte[] key,
+ byte[] iv) throws IOException;
+
+ /** Test crypto reading with different buffer size. */
+ @Test(timeout=12)
+ public void testRead() throws Exception {
+OutputStream out = getOutputStream(defaultBufferSize);
+writeData(out);
+
+// Default buffer size
+InputStream in = getInputStream(defaultBufferSize);
+readCheck(in)
[05/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8ca1b27..076c9c8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension
+.EncryptedKeyVersion;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
@@ -107,6 +110,8 @@ import java.io.StringWriter;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -120,6 +125,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -135,12 +141,17 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
@@ -165,6 +176,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.UnknownCipherSuiteException;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -176,6 +188,8 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -317,7 +331,7 @@ public class FSNamesystem implements Namesystem,
FSClusterStats,
private HdfsFileStatus getAuditFileInfo(String path, boolean resolveSymlink)
throws IOException {
return (isAuditEnabled() && isExternalInvocation())
-? dir.getFileInfo(path, resolveSymlink) : null;
+? dir.getFileInfo(path, resolveSymlink, false) : null;
}
private void logAuditEvent(boolean succeeded, String cmd, String src)
@@ -403,6 +417,8 @@ public class FSNamesystem implements Namesystem,
FSClusterStats,
private final CacheManager cacheManager;
private final DatanodeStatistics datanodeStatistics;
+ private String nameserviceId;
+
private RollingUpgradeInfo rollingUpgradeInfo = null;
/**
* A flag that indicates whether the checkpointer should checkpoint a
rollback
@@ -519,6 +535,11 @@ public class FSNamesystem implements Namesystem,
FSClusterStats,
private final NNConf nnConf;
+ private KeyProviderCryptoExtension provider = null;
+ private KeyProvider.Options providerOptions = null;
+
+ private final CryptoCodec codec;
+
private volatile boolean imageLoaded = false;
private final Condition cond;
@@ -738,6 +759,14 @@ public class FSNamesystem implements Namesy
[02/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
index 41b381a..3e8de4f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
@@ -191,6 +191,26 @@ $H3 Update and Overwrite
If `-update` is used, `1` is overwritten as well.
+$H3 raw Namespace Extended Attribute Preservation
+
+ This section only applies to HDFS.
+
+ If the target and all of the source pathnames are in the /.reserved/raw
+ hierarchy, then 'raw' namespace extended attributes will be preserved.
+ 'raw' xattrs are used by the system for internal functions such as encryption
+ meta data. They are only visible to users when accessed through the
+ /.reserved/raw hierarchy.
+
+ raw xattrs are preserved based solely on whether /.reserved/raw prefixes are
+ supplied. The -p (preserve, see below) flag does not impact preservation of
+ raw xattrs.
+
+ To prevent raw xattrs from being preserved, simply do not use the
+ /.reserved/raw prefix on any of the source and target paths.
+
+ If the /.reserved/raw prefix is specified on only a subset of the source and
+ target paths, an error will be displayed and a non-0 exit code returned.
+
Command Line Options
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
index 1aea500..c5ab420 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
@@ -24,14 +24,16 @@ import static org.mockito.Mockito.doAnswer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -51,10 +53,16 @@ import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.mapreduce.CryptoUtils;
import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl;
+import
org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
@@ -63,40 +71,48 @@ import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import com.google.common.collect.Lists;
+
public class TestMerger {
private Configuration conf;
private JobConf jobConf;
private FileSystem fs;
-
+
@Before
public void setup() throws IOException {
conf = new Configuration();
jobConf = new JobConf();
fs = FileSystem.getLocal(conf);
}
-
- @After
- public void cleanup() throws IOException {
-fs.delete(new Path(jobConf.getLocalDirs()[0]), true);
+
+
+ @Test
+ public void testEncryptedMerger() throws Throwable {
+jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
+conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true);
+Credentials credentials =
UserGroupInformation.getCurrentUser().getCrede
[11/11] git commit: Fix up CHANGES.txt for HDFS-6134, HADOOP-10150 and related JIRAs following merge to branch-2
wang) + + HDFS-6391. Get the Key/IV from the NameNode for encrypted files in + DFSClient. (Charles Lamb and wang) + + HDFS-6389. Rename restrictions for encryption zones. (clamb) + + HDFS-6605. Client server negotiation of cipher suite. (wang) + + HDFS-6625. Remove the Delete Encryption Zone function (clamb) + + HDFS-6516. List of Encryption Zones should be based on inodes (clamb) + + HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao) + + HDFS-6635. Refactor encryption zone functionality into new + EncryptionZoneManager class. (wang) + + HDFS-6474. Namenode needs to get the actual keys and iv from the + KeyProvider. (wang) + + HDFS-6619. Clean up encryption-related tests. (wang) + + HDFS-6405. Test Crypto streams in HDFS. (yliu via wang) + + HDFS-6490. Fix the keyid format for generated keys in + FSNamesystem.createEncryptionZone (clamb) + + HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode. + (wang) + + HDFS-6718. Remove EncryptionZoneManager lock. (wang) + + HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang) + + HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in + EZManager#createEncryptionZone. (clamb) + + HDFS-6724. Decrypt EDEK before creating + CryptoInputStream/CryptoOutputStream. (wang) + + HDFS-6509. Create a special /.reserved/raw directory for raw access to + encrypted data. (clamb via wang) + + HDFS-6771. Require specification of an encryption key when creating + an encryption zone. (wang) + + HDFS-6730. Create a .RAW extended attribute namespace. (clamb) + + HDFS-6692. Add more HDFS encryption tests. (wang) + + HDFS-6780. Batch the encryption zones listing API. (wang) + + HDFS-6394. HDFS encryption documentation. (wang) + + HDFS-6834. Improve the configuration guidance in DFSClient when there + are no Codec classes found in configs. (umamahesh) + + HDFS-6546. Add non-superuser capability to get the encryption zone + for a specific path. (clamb) + + HDFS-6733. Creating encryption zone results in NPE when + KeyProvider is null. (clamb) + + HDFS-6785. Should not be able to create encryption zone using path + to a non-directory file. (clamb) + + HDFS-6807. Fix TestReservedRawPaths. (clamb) + + HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured + as boolean. (umamahesh) + + HDFS-6817. Fix findbugs and other warnings. (yliu) + + HDFS-6839. Fix TestCLI to expect new output. (clamb) +-- + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c37f02e/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6f07104..387d2cc 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -102,6 +102,17 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5885. build/test/test.mapred.spill causes release audit warnings (Chen He via jlowe) +BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS + + MAPREDUCE-5890. Support for encrypting Intermediate + data and spills in local filesystem. (asuresh via tucu) + + MAPREDUCE-6007. Add support to distcp to preserve raw.* namespace + extended attributes. (clamb) + + MAPREDUCE-6041. Fix TestOptionsParser. (clamb) +-- + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES @@ -114,7 +125,7 @@ Release 2.5.1 - UNRELEASED BUG FIXES -MAPREDUCE-6033. Updated access check for displaying job information +MAPREDUCE-6033. Updated access check for displaying job information (Yu Gao via Eric Yang) Release 2.5.0 - 2014-08-11
[10/11] git commit: HDFS-6134 and HADOOP-10150 subtasks.
HDFS-6134 and HADOOP-10150 subtasks. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c77bd85b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c77bd85b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c77bd85b Branch: refs/heads/branch-2 Commit: c77bd85b621e23738855628230bf8db1bc5d007d Parents: 631dea8 Author: Alejandro Abdelnur Authored: Tue Aug 26 10:38:28 2014 -0700 Committer: Alejandro Abdelnur Committed: Thu Aug 28 15:03:08 2014 -0700 -- BUILDING.txt| 21 + .../hadoop-common/CHANGES-fs-encryption.txt | 61 ++ hadoop-common-project/hadoop-common/pom.xml | 19 +- .../hadoop-common/src/CMakeLists.txt| 34 + .../hadoop-common/src/config.h.cmake| 1 + .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 ++ .../org/apache/hadoop/crypto/CipherSuite.java | 115 +++ .../org/apache/hadoop/crypto/CryptoCodec.java | 174 + .../apache/hadoop/crypto/CryptoInputStream.java | 680 + .../hadoop/crypto/CryptoOutputStream.java | 286 +++ .../apache/hadoop/crypto/CryptoStreamUtils.java | 70 ++ .../org/apache/hadoop/crypto/Decryptor.java | 72 ++ .../org/apache/hadoop/crypto/Encryptor.java | 71 ++ .../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 .../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 .../org/apache/hadoop/crypto/OpensslCipher.java | 287 +++ .../crypto/random/OpensslSecureRandom.java | 119 +++ .../hadoop/crypto/random/OsSecureRandom.java| 115 +++ .../hadoop/fs/CommonConfigurationKeys.java | 1 - .../fs/CommonConfigurationKeysPublic.java | 30 + .../apache/hadoop/fs/FSDataOutputStream.java| 2 +- .../apache/hadoop/fs/FileEncryptionInfo.java| 102 +++ .../fs/crypto/CryptoFSDataInputStream.java | 37 + .../fs/crypto/CryptoFSDataOutputStream.java | 47 ++ .../hadoop/fs/shell/CommandWithDestination.java | 75 +- .../apache/hadoop/fs/shell/CopyCommands.java| 6 +- .../apache/hadoop/util/NativeCodeLoader.java| 5 + .../hadoop/util/NativeLibraryChecker.java | 21 +- .../org/apache/hadoop/crypto/OpensslCipher.c| 382 ++ .../hadoop/crypto/org_apache_hadoop_crypto.h| 61 ++ .../hadoop/crypto/random/OpensslSecureRandom.c | 335 .../random/org_apache_hadoop_crypto_random.h| 40 + .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../src/main/resources/core-default.xml | 69 ++ .../src/site/apt/FileSystemShell.apt.vm | 11 +- .../hadoop/crypto/CryptoStreamsTestBase.java| 721 ++ .../apache/hadoop/crypto/TestCryptoCodec.java | 186 + .../apache/hadoop/crypto/TestCryptoStreams.java | 381 ++ .../crypto/TestCryptoStreamsForLocalFS.java | 120 +++ .../hadoop/crypto/TestCryptoStreamsNormal.java | 123 +++ ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 31 + .../apache/hadoop/crypto/TestOpensslCipher.java | 110 +++ .../crypto/random/TestOpensslSecureRandom.java | 114 +++ .../crypto/random/TestOsSecureRandom.java | 139 .../hadoop/util/TestNativeCodeLoader.java | 4 + .../src/test/resources/testConf.xml | 18 +- .../hadoop-hdfs/CHANGES-fs-encryption.txt | 102 +++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../hadoop-hdfs/src/main/bin/hdfs | 5 +- .../main/java/org/apache/hadoop/fs/Hdfs.java| 20 +- .../main/java/org/apache/hadoop/fs/XAttr.java | 13 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 158 +++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 32 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java| 38 + .../hadoop/hdfs/DistributedFileSystem.java | 52 +- .../hdfs/UnknownCipherSuiteException.java | 38 + .../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java| 50 ++ .../hadoop/hdfs/client/HdfsDataInputStream.java | 38 +- .../hdfs/client/HdfsDataOutputStream.java | 36 +- .../hadoop/hdfs/protocol/ClientProtocol.java| 30 +- .../hadoop/hdfs/protocol/EncryptionZone.java| 79 ++ .../hdfs/protocol/EncryptionZoneIterator.java | 51 ++ .../hdfs/protocol/EncryptionZoneWithId.java | 64 ++ .../protocol/EncryptionZoneWithIdIterator.java | 53 ++ .../hadoop/hdfs/protocol/HdfsFileStatus.java| 15 +- .../hdfs/protocol/HdfsLocatedFileStatus.java| 8 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 23 +- .../protocol/SnapshottableDirectoryStatus.java | 2 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 56 +- .../ClientNamenodeProtocolTranslatorPB.java | 81 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 105 ++- .../server/blockmanageme
[06/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
new file mode 100644
index 000..e45d540
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -0,0 +1,296 @@
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.XAttrHelper;
+import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants
+.CRYPTO_XATTR_ENCRYPTION_ZONE;
+
+/**
+ * Manages the list of encryption zones in the filesystem.
+ *
+ * The EncryptionZoneManager has its own lock, but relies on the FSDirectory
+ * lock being held for many operations. The FSDirectory lock should not be
+ * taken if the manager lock is already held.
+ */
+public class EncryptionZoneManager {
+
+ public static Logger LOG = LoggerFactory.getLogger(EncryptionZoneManager
+ .class);
+
+ private static final EncryptionZoneWithId NULL_EZ =
+ new EncryptionZoneWithId("", "", -1);
+
+ /**
+ * EncryptionZoneInt is the internal representation of an encryption zone.
The
+ * external representation of an EZ is embodied in an EncryptionZone and
+ * contains the EZ's pathname.
+ */
+ private static class EncryptionZoneInt {
+private final String keyName;
+private final long inodeId;
+
+EncryptionZoneInt(long inodeId, String keyName) {
+ this.keyName = keyName;
+ this.inodeId = inodeId;
+}
+
+String getKeyName() {
+ return keyName;
+}
+
+long getINodeId() {
+ return inodeId;
+}
+ }
+
+ private final TreeMap encryptionZones;
+ private final FSDirectory dir;
+ private final int maxListEncryptionZonesResponses;
+
+ /**
+ * Construct a new EncryptionZoneManager.
+ *
+ * @param dir Enclosing FSDirectory
+ */
+ public EncryptionZoneManager(FSDirectory dir, Configuration conf) {
+this.dir = dir;
+encryptionZones = new TreeMap();
+maxListEncryptionZonesResponses = conf.getInt(
+DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
+DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT
+);
+Preconditions.checkArgument(maxListEncryptionZonesResponses >= 0,
+DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES + " " +
+"must be a positive integer."
+);
+ }
+
+ /**
+ * Add a new encryption zone.
+ *
+ * Called while holding the FSDirectory lock.
+ *
+ * @param inodeId of the encryption zone
+ * @param keyName encryption zone key name
+ */
+ void addEncryptionZone(Long inodeId, String keyName) {
+assert dir.hasWriteLock();
+final EncryptionZoneInt ez = new EncryptionZoneInt(inodeId, keyName);
+encryptionZones.put(inodeId, ez);
+ }
+
+ /**
+ * Remove an encryption zone.
+ *
+ * Called while holding the FSDirectory lock.
+ */
+ void removeEncryptionZone(Long inodeId) {
+assert dir.hasWriteLock();
+encryptionZones.remove(inodeId);
+ }
+
+ /**
+ * Returns true if an IIP is within an encryption zone.
+ *
+ * Called while holding the FSDirectory lock.
+ */
+ boolean isInAnEZ(INodesInPath iip)
+ throws UnresolvedLinkException, SnapshotAccessControlException {
+assert dir.hasReadLock();
+return (getEncryptionZoneForPath(iip) != null);
+ }
+
+ /**
+ * Returns the path of the EncryptionZoneInt.
+ *
+ * Called while holding the FSDirectory lock.
+ */
+ private String getFullPathName(EncryptionZoneInt ezi) {
+assert dir.hasReadLock();
+return dir.getInode(ezi.getINodeId()).getFullPathName();
+ }
+
+ /**
+ * Get the key name for an encryption zone. Returns null if iip is
+ * not within an encryption zone.
+ *
+ * Called while holding the FSDirectory lock.
+ */
+ String getKeyName(final INodesInPath iip) {
+assert dir.hasReadLock();
+EncryptionZoneI
[03/11] HDFS-6134 and HADOOP-10150 subtasks.
x27;.'
@@ -126,6 +126,42 @@
+ setfattr : Add an xattr of raw namespace
+
+ -fs NAMENODE -touchz /file1
+ -fs NAMENODE -setfattr -n raw.a1 -v 123456 /file1
+
+
+ -fs NAMENODE -rm /file1
+
+
+
+ SubstringComparator
+ setfattr: User doesn't have permission for
xattr: raw.a1
+
+
+
+
+
+
+setfattr : Add an xattr of raw namespace
+
+-fs NAMENODE -touchz /file1
+-fs NAMENODE -setfattr -n raw.a1 -v 123456
/.reserved/raw/file1
+-fs NAMENODE -getfattr -n raw.a1
/.reserved/raw/file1
+
+
+-fs NAMENODE -rm /file1
+
+
+
+SubstringComparator
+raw.a1="123456"
+
+
+
+
+
setfattr : Add an xattr, and encode is text
-fs NAMENODE -touchz /file1
@@ -256,6 +292,26 @@
+
+
+setfattr : Remove an xattr of raw namespace
+
+-fs NAMENODE -touchz /file1
+-fs NAMENODE -setfattr -n raw.a1 -v 123456
/.reserved/raw/file1
+-fs NAMENODE -setfattr -n raw.a2 -v 123456
/.reserved/raw/file1
+-fs NAMENODE -setfattr -x raw.a2
/.reserved/raw/file1
+-fs NAMENODE -getfattr -d /.reserved/raw/file1
+
+
+-fs NAMENODE -rm /file1
+
+
+
+SubstringComparator
+ # file:
/.reserved/raw/file1#LF#raw.a1="123456"#LF#
+
+
+
getfattr : Get an xattr
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-mapreduce-project/CHANGES-fs-encryption.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES-fs-encryption.txt
b/hadoop-mapreduce-project/CHANGES-fs-encryption.txt
new file mode 100644
index 000..3e1718e
--- /dev/null
+++ b/hadoop-mapreduce-project/CHANGES-fs-encryption.txt
@@ -0,0 +1,20 @@
+Hadoop MapReduce Change Log
+
+fs-encryption (Unreleased)
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+MAPREDUCE-5890. Support for encrypting Intermediate
+data and spills in local filesystem. (asuresh via tucu)
+
+ IMPROVEMENTS
+
+MAPREDUCE-6007. Add support to distcp to preserve raw.* namespace
+extended attributes. (clamb)
+
+HDFS-6872. Fix TestOptionsParser. (clamb)
+
+ BUG FIXES
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
index cfcf0f2..be7fe18 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
@@ -43,6 +44,7 @@ import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.CryptoUtils;
/**
* BackupStore is an utility class that is used to support
@@ -572,7 +574,9 @@ public class BackupStore {
file = lDirAlloc.getLocalPathForWrite(tmp.toUri().getPath(),
-1, conf);
- return new Writer(conf, fs, file);
+ FSDataOutputStream out = fs.create(file);
+ out = CryptoUtils.wrapIfNecessary(conf, out);
+ return new Writer(conf, out, null, null, null, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-ma
[01/11] git commit: HADOOP-8815. RandomDatum needs to override hashCode(). Contributed by Brandon Li.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b7367dc6a -> 5c37f02e2
HADOOP-8815. RandomDatum needs to override hashCode(). Contributed by Brandon
Li.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1389661
13f79535-47bb-0310-9956-ffa450edef68
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
(cherry picked from commit 3ede27f4557c9e90430a7a3f385b8be243e89688)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/631dea88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/631dea88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/631dea88
Branch: refs/heads/branch-2
Commit: 631dea88d8a89f03e1643b2c9179c775ee4112f2
Parents: b7367dc
Author: Suresh Srinivas
Authored: Tue Sep 25 00:11:56 2012 +
Committer: Alejandro Abdelnur
Committed: Thu Aug 28 15:02:57 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/io/RandomDatum.java | 6 ++
.../java/org/apache/hadoop/io/compress/TestCodec.java| 11 +++
3 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/631dea88/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f79c1fe..54f8dad 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -321,6 +321,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10989. Work around buggy getgrouplist() implementations on Linux
that
return 0 on failure. (cnauroth)
+HADOOP-8815. RandomDatum needs to override hashCode().
+(Brandon Li via suresh)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/631dea88/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/RandomDatum.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/RandomDatum.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/RandomDatum.java
index 8f99aab..01e00b7 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/RandomDatum.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/RandomDatum.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.io;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Random;
@@ -65,6 +66,11 @@ public class RandomDatum implements
WritableComparable {
return compareTo((RandomDatum)o) == 0;
}
+ @Override
+ public int hashCode() {
+return Arrays.hashCode(this.data);
+ }
+
private static final char[] HEX_DIGITS =
{'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/631dea88/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index fe533ff..54768f3 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -34,6 +34,8 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Random;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -226,6 +228,15 @@ public class TestCodec {
v2.readFields(inflateIn);
assertTrue("original and compressed-then-decompressed-output not equal",
k1.equals(k2) && v1.equals(v2));
+
+ // original and compressed-then-decompressed-output have the same
hashCode
+ Map m = new HashMap();
+ m.put(k1, k1.toString());
+ m.put(v1, v1.toString());
+ String result = m.get(k2);
+ assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
+ result = m.get(v2);
+ assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
}
// De-compress data byte-at-a-time
git commit: HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 fc99a6b80 -> b7367dc6a
HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header.
(tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7367dc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7367dc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7367dc6
Branch: refs/heads/branch-2
Commit: b7367dc6a29bd70648f748007e425baa203985a8
Parents: fc99a6b
Author: Alejandro Abdelnur
Authored: Thu Aug 28 14:45:40 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Aug 28 14:47:23 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../web/DelegationTokenAuthenticatedURL.java| 81
.../DelegationTokenAuthenticationHandler.java | 14 +++-
.../web/DelegationTokenAuthenticator.java | 19 -
...tionTokenAuthenticationHandlerWithMocks.java | 46 ++-
.../delegation/web/TestWebDelegationToken.java | 50 +++-
6 files changed, 187 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7367dc6/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index e20c5ff..f79c1fe 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -137,6 +137,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10998. Fix bash tab completion code to work (Jim Hester via aw)
+HADOOP-10880. Move HTTP delegation tokens out of URL querystring to
+a header. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7367dc6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index d955ada..5aeb177 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -125,6 +125,8 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
}
}
+ private boolean useQueryStringforDelegationToken = false;
+
/**
* Creates an DelegationTokenAuthenticatedURL.
*
@@ -171,6 +173,34 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
}
/**
+ * Sets if delegation token should be transmitted in the URL query string.
+ * By default it is transmitted using the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ *
+ * This method is provided to enable WebHDFS backwards compatibility.
+ *
+ * @param useQueryString TRUE if the token is transmitted in
the
+ * URL query string, FALSE if the delegation token is
transmitted
+ * using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER}
HTTP
+ * header.
+ */
+ @Deprecated
+ protected void setUseQueryStringForDelegationToken(boolean useQueryString) {
+useQueryStringforDelegationToken = useQueryString;
+ }
+
+ /**
+ * Returns if delegation token is transmitted as a HTTP header.
+ *
+ * @return TRUE if the token is transmitted in the URL query
+ * string, FALSE if the delegation token is transmitted using
the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ */
+ public boolean useQueryStringForDelegationToken() {
+return useQueryStringforDelegationToken;
+ }
+
+ /**
* Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
* Token only if the given auth token is an instance of {@link Token} and
* it contains a Delegation Token, otherwise use the configured
@@ -235,23 +265,41 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
* @throws IOException if an IO error occurred.
* @throws AuthenticationException if an authentication exception occurred.
*/
+ @SuppressWarnings("unchecked")
public HttpURLConnection openConnection(URL url, Token token, String doAs)
throws IOException, AuthenticationException {
Preconditions.checkNotN
git commit: HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c4c9a7841 -> d1ae479aa
HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header.
(tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1ae479a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1ae479a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1ae479a
Branch: refs/heads/trunk
Commit: d1ae479aa5ae4d3e7ec80e35892e1699c378f813
Parents: c4c9a78
Author: Alejandro Abdelnur
Authored: Thu Aug 28 14:45:40 2014 -0700
Committer: Alejandro Abdelnur
Committed: Thu Aug 28 14:45:40 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../web/DelegationTokenAuthenticatedURL.java| 81
.../DelegationTokenAuthenticationHandler.java | 14 +++-
.../web/DelegationTokenAuthenticator.java | 19 -
...tionTokenAuthenticationHandlerWithMocks.java | 46 ++-
.../delegation/web/TestWebDelegationToken.java | 50 +++-
6 files changed, 187 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ecbaaab..641635b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10998. Fix bash tab completion code to work (Jim Hester via aw)
+HADOOP-10880. Move HTTP delegation tokens out of URL querystring to
+a header. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index d955ada..5aeb177 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -125,6 +125,8 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
}
}
+ private boolean useQueryStringforDelegationToken = false;
+
/**
* Creates an DelegationTokenAuthenticatedURL.
*
@@ -171,6 +173,34 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
}
/**
+ * Sets if delegation token should be transmitted in the URL query string.
+ * By default it is transmitted using the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ *
+ * This method is provided to enable WebHDFS backwards compatibility.
+ *
+ * @param useQueryString TRUE if the token is transmitted in
the
+ * URL query string, FALSE if the delegation token is
transmitted
+ * using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER}
HTTP
+ * header.
+ */
+ @Deprecated
+ protected void setUseQueryStringForDelegationToken(boolean useQueryString) {
+useQueryStringforDelegationToken = useQueryString;
+ }
+
+ /**
+ * Returns if delegation token is transmitted as a HTTP header.
+ *
+ * @return TRUE if the token is transmitted in the URL query
+ * string, FALSE if the delegation token is transmitted using
the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ */
+ public boolean useQueryStringForDelegationToken() {
+return useQueryStringforDelegationToken;
+ }
+
+ /**
* Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
* Token only if the given auth token is an instance of {@link Token} and
* it contains a Delegation Token, otherwise use the configured
@@ -235,23 +265,41 @@ public class DelegationTokenAuthenticatedURL extends
AuthenticatedURL {
* @throws IOException if an IO error occurred.
* @throws AuthenticationException if an authentication exception occurred.
*/
+ @SuppressWarnings("unchecked")
public HttpURLConnection openConnection(URL url, Token token, String doAs)
throws IOException, AuthenticationException {
Preconditions.checkNotN
svn commit: r1619556 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tucu Date: Thu Aug 21 19:03:28 2014 New Revision: 1619556 URL: http://svn.apache.org/r1619556 Log: HADOOP-10992. Merge KMS to branch-2, updating hadoop-common CHANGES.txt. (tucu) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619556&r1=1619555&r2=1619556&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 19:03:28 2014 @@ -13,8 +13,6 @@ Trunk (Unreleased) NEW FEATURES -HADOOP-10433. Key Management Server based on KeyProvider API. (tucu) - HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop. (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao, Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys, @@ -25,9 +23,6 @@ Trunk (Unreleased) Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys, Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth) -HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey -methods to KeyProvider. (asuresh via tucu) - IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution @@ -121,93 +116,15 @@ Trunk (Unreleased) HADOOP-9833 move slf4j to version 1.7.5 (Kousuke Saruta via stevel) -HADOOP-10141. Create KeyProvider API to separate encryption key storage -from the applications. (omalley) - -HADOOP-10201. Add listing to KeyProvider API. (Larry McCay via omalley) - -HADOOP-10177. Create CLI tools for managing keys. (Larry McCay via omalley) - -HADOOP-10244. TestKeyShell improperly tests the results of delete (Larry -McCay via omalley) - HADOOP-10325. Improve jenkins javadoc warnings from test-patch.sh (cmccabe) HADOOP-10342. Add a new method to UGI to use a Kerberos login subject to build a new UGI. (Larry McCay via omalley) -HADOOP-10237. JavaKeyStoreProvider needs to set keystore permissions -correctly. (Larry McCay via omalley) - -HADOOP-10432. Refactor SSLFactory to expose static method to determine -HostnameVerifier. (tucu) - -HADOOP-10427. KeyProvider implementations should be thread safe. (tucu) - -HADOOP-10429. KeyStores should have methods to generate the materials -themselves, KeyShell should use them. (tucu) - -HADOOP-10428. JavaKeyStoreProvider should accept keystore password via -configuration falling back to ENV VAR. (tucu) - -HADOOP-10430. KeyProvider Metadata should have an optional description, -there should be a method to retrieve the metadata from all keys. (tucu) - -HADOOP-10534. KeyProvider getKeysMetadata should take a list of names -rather than returning all keys. (omalley) - HADOOP-10563. Remove the dependency of jsp in trunk. (wheat9) HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9) -HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata. -(tucu) - -HADOOP-10695. KMSClientProvider should respect a configurable timeout. -(yoderme via tucu) - -HADOOP-10757. KeyProvider KeyVersion should provide the key name. -(asuresh via tucu) - -HADOOP-10769. Create KeyProvider extension to handle delegation tokens. -(Arun Suresh via atm) - -HADOOP-10812. Delegate KeyProviderExtension#toString to underlying -KeyProvider. (wang) - -HADOOP-10736. Add key attributes to the key shell. (Mike Yoder via wang) - -HADOOP-10824. Refactor KMSACLs to avoid locking. (Benoy Antony via umamahesh) - -HADOOP-10841. EncryptedKeyVersion should have a key name property. -(asuresh via tucu) - -HADOOP-10842. CryptoExtension generateEncryptedKey method should -receive the key name. (asuresh via tucu) - -HADOOP-10750. KMSKeyProviderCache should be in hadoop-common. -(asuresh via tucu) - -HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey -in the REST API. (asuresh via tucu) - -HADOOP-10891. Add EncryptedKeyVersion factory method to -KeyProviderCryptoExtension. (wang) - -HADOOP-10756. KMS audit log should consolidate successful similar requests. -(asuresh via tucu) - -HADOOP-10793. KeyShell args should use single-dash style. (wang) - -HADOOP-10936. Change default KeyProvider bitlength to 128. (wang) - -HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting -underlying store. (asuresh via tucu) - -HADOOP-10770. KMS add delegation token support. (tucu) - -HADOOP-10698. KMS, add proxyuser support. (tucu) - BUG FIXES HADOOP-9451. Fault single-layer config if node group
svn commit: r1619550 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/ hadoop-common/src/main/java/org/apache/hadoop
Author: tucu Date: Thu Aug 21 19:00:01 2014 New Revision: 1619550 URL: http://svn.apache.org/r1619550 Log: HADOOP-10770. KMS add delegation token support. (tucu) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619550&r1=1619549&r2=1619550&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 19:00:01 2014 @@ -125,6 +125,8 @@ Release 2.6.0 - UNRELEASED HADOOP-10881. Clarify usage of encryption and encrypted encryption key in KeyProviderCryptoExtension. (wang) +HADOOP-10770. KMS add delegation token support. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java?rev=1619550&r1=1619549&r2=1619550&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java Thu Aug 21 19:00:01 2014 @@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import java.io.IOException; + /** * A KeyProvider extension with the ability to add a renewer's Delegation * Tokens to the provided Credentials. @@ -45,9 +47,10 @@ public class KeyProviderDelegationTokenE * @param renewer the user allowed to renew the delegation tokens * @param credentials cache in which to add new delegation tokens * @return list of new delegation tokens + * @throws IOException thrown if IOException if an IO error occurs. */ public Token[] addDelegationTokens(final String renewer, -Credentials credentials); +Credentials credentials) throws IOException; } /** @@ -76,9 +79,10 @@ public class KeyProviderDelegationTokenE * @param renewer the user allowed to renew the delegation tokens * @param credentials cache in which to add new delegation tokens * @return list of new delegation tokens + * @throws IOException thrown if IOException if an IO error o
svn commit: r1619541 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/ hadoop-kms/src/site/apt/ hadoop-kms/s
Author: tucu
Date: Thu Aug 21 18:59:41 2014
New Revision: 1619541
URL: http://svn.apache.org/r1619541
Log:
HADOOP-10756. KMS audit log should consolidate successful similar requests.
(asuresh via tucu)
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619541&r1=1619540&r2=1619541&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:41 2014
@@ -163,6 +163,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10891. Add EncryptedKeyVersion factory method to
KeyProviderCryptoExtension. (wang)
+HADOOP-10756. KMS audit log should consolidate successful similar requests.
+(asuresh via tucu)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java?rev=1619541&r1=1619540&r2=1619541&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
Thu Aug 21 18:59:41 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
@@ -27,7 +28,6 @@ import org.apache.hadoop.security.Access
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
-import org.apache.hadoop.util.StringUtils;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
@@ -59,22 +59,25 @@ import java.util.Map;
@Path(KMSRESTConstants.SERVICE_VERSION)
@InterfaceAudience.Private
public class KMS {
- private static final String CREATE_KEY = "CREATE_KEY";
- private static final String DELETE_KEY = "DELETE_KEY";
- private static final String ROLL_NEW_VERSION = "ROLL_NEW_VERSION";
- private static final String GET_KEYS = "GET_KEYS";
- private static final String GET_KEYS_METADATA = "GET_KEYS_METADATA";
- private static final String GET_KEY_VERSION = "GET_KEY_VERSION";
- private static final String GET_CURRENT_KEY = "GET_CURRENT_KEY";
- private static final String GET_KEY_VERSIONS = "GET_KEY_VERSIONS";
- private static final String GET_METADATA = "GET_METADATA";
- private static final String GENERATE_EEK = "GENERATE_EEK";
- private static final String DECRYPT_EEK = "DECRYPT_EEK";
+ public static final Str
svn commit: r1619548 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoop-kms/src/main/java/org/apache/hadoo
Author: tucu
Date: Thu Aug 21 18:59:54 2014
New Revision: 1619548
URL: http://svn.apache.org/r1619548
Log:
HADOOP-10862. Miscellaneous trivial corrections to KMS classes. (asuresh via
tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619548&r1=1619547&r2=1619548&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:54 2014
@@ -292,6 +292,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10939. Fix TestKeyProviderFactory testcases to use default 128 bit
length keys. (Arun Suresh via wang)
+HADOOP-10862. Miscellaneous trivial corrections to KMS classes.
+(asuresh via tucu)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1619548&r1=1619547&r2=1619548&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
Thu Aug 21 18:59:54 2014
@@ -512,7 +512,7 @@ public class KMSClientProvider extends K
List batch = new ArrayList();
int batchLen = 0;
for (String name : keyNames) {
- int additionalLen = KMSRESTConstants.KEY_OP.length() + 1 + name.length();
+ int additionalLen = KMSRESTConstants.KEY.length() + 1 + name.length();
batchLen += additionalLen;
// topping at 1500 to account for initial URL and encoded names
if (batchLen > 1500) {
@@ -536,7 +536,7 @@ public class KMSClientProvider extends K
for (String[] keySet : keySets) {
if (keyNames.length > 0) {
Map queryStr = new HashMap();
-queryStr.put(KMSRESTConstants.KEY_OP, keySet);
+queryStr.put(KMSRESTConstants.KEY, keySet);
URL url = createURL(KMSRESTConstants.KEYS_METADATA_RESOURCE, null,
null, queryStr);
HttpURLConnection conn = createConnection(url, HTTP_GET);
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java?rev=1619548&r1=1619547&r2=1619548&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
Thu Aug 21 18:59:54 2014
@@ -37,7 +37,7 @@ public class KMSRESTConstants {
public static final String EEK_SUB_RESOURCE = "_eek";
public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion";
- public static final String KEY_OP = "key";
+ public static final String KEY = "key&quo
svn commit: r1619554 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Author: tucu
Date: Thu Aug 21 19:00:08 2014
New Revision: 1619554
URL: http://svn.apache.org/r1619554
Log:
HADOOP-10488. TestKeyProviderFactory fails randomly. (tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619554&r1=1619553&r2=1619554&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 19:00:08 2014
@@ -303,6 +303,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey
performance. (hitliuyi via tucu)
+HADOOP-10488. TestKeyProviderFactory fails randomly. (tucu)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1619554&r1=1619553&r2=1619554&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Thu Aug 21 19:00:08 2014
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.List;
+import java.util.UUID;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.fs.FileStatus;
@@ -32,6 +34,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
@@ -40,8 +43,14 @@ import static org.junit.Assert.assertTru
public class TestKeyProviderFactory {
- private static final File tmpDir =
- new File(System.getProperty("test.build.data", "/tmp"), "key");
+ private static File tmpDir;
+
+ @Before
+ public void setup() {
+tmpDir = new File(System.getProperty("test.build.data", "target"),
+UUID.randomUUID().toString());
+tmpDir.mkdirs();
+ }
@Test
public void testFactory() throws Exception {
svn commit: r1619549 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java src/test/java/org/apach
Author: tucu
Date: Thu Aug 21 18:59:56 2014
New Revision: 1619549
URL: http://svn.apache.org/r1619549
Log:
HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting underlying
store. (asuresh via tucu)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619549&r1=1619548&r2=1619549&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:56 2014
@@ -170,6 +170,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10936. Change default KeyProvider bitlength to 128. (wang)
+HADOOP-10224. JavaKeyStoreProvider has to protect against corrupting
+underlying store. (asuresh via tucu)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1619549&r1=1619548&r2=1619549&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
Thu Aug 21 18:59:56 2014
@@ -27,8 +27,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.crypto.spec.SecretKeySpec;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
@@ -80,6 +83,9 @@ import java.util.concurrent.locks.Reentr
@InterfaceAudience.Private
public class JavaKeyStoreProvider extends KeyProvider {
private static final String KEY_METADATA = "KeyMetadata";
+ private static Logger LOG =
+ LoggerFactory.getLogger(JavaKeyStoreProvider.class);
+
public static final String SCHEME_NAME = "jceks";
public static final String KEYSTORE_PASSWORD_FILE_KEY =
@@ -115,6 +121,10 @@ public class JavaKeyStoreProvider extend
if (pwFile != null) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL pwdFile = cl.getResource(pwFile);
+if (pwdFile == null) {
+ // Provided Password file does not exist
+ throw new IOException("Password file does not exists");
+}
if (pwdFile != null) {
InputStream is = pwdFile.openStream();
try {
@@ -129,19 +139,25 @@ public class JavaKeyStoreProvider extend
password = KEYSTORE_PASSWORD_DEFAULT;
}
try {
+ Path oldPath = constructOldPath(path);
+ Path newPath = constructNewPath(path);
keyStore = KeyStore.getInstance(SCHEME_NAME);
+ FsPermission perm = null;
if (fs.exists(path)) {
-// save off permissions in case we need to
-// rewrite the keystore in flush()
-FileStatus s = fs.getFileStatus(path);
-permissions = s.getPermission();
-
-keyStore.load(fs.open(path), password);
+// flush did not proceed to completion
+// _NEW should not exist
+if (fs.exists(newPath)) {
+ throw new IOException(
+ String.format("Keystore not loaded due to some inconsistency "
+ + "('%s' and '%s' should not exist together)!!", path, newPath));
+}
+perm = tryLoadFromPath(path, oldPath);
} else {
-permissions = new FsPermission("700");
-// required to create an empty keystore. *sigh*
-keyStore.load(null, password);
+perm = tryLoadIncompleteFlush(oldPath, newPath);
}
+ // Need to save off permissions in case we need to
+ // rewrite the keystore in flush()
+ permissions = perm;
} catch (KeyStoreException e) {
throw new IOException(&qu
svn commit: r1619553 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tucu Date: Thu Aug 21 19:00:07 2014 New Revision: 1619553 URL: http://svn.apache.org/r1619553 Log: Fix hadoop-common CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619553&r1=1619552&r2=1619553&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 19:00:07 2014 @@ -185,8 +185,6 @@ Release 2.6.0 - UNRELEASED HADOOP-10507. FsShell setfacl can throw ArrayIndexOutOfBoundsException when no perm is specified. (Stephen Chu and Sathish Gurram via cnauroth) - BUG FIXES - HADOOP-10780. hadoop_user_info_alloc fails on FreeBSD due to incorrect sysconf use (Dmitry Sivachenko via Colin Patrick McCabe)
svn commit: r1619552 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoop-kms/src/main/java/org/apache/hadoo
Author: tucu
Date: Thu Aug 21 19:00:06 2014
New Revision: 1619552
URL: http://svn.apache.org/r1619552
Log:
HADOOP-10698. KMS, add proxyuser support. (tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619552&r1=1619551&r2=1619552&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 19:00:06 2014
@@ -127,6 +127,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10770. KMS add delegation token support. (tucu)
+HADOOP-10698. KMS, add proxyuser support. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1619552&r1=1619551&r2=1619552&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
Thu Aug 21 19:00:06 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -52,6 +53,7 @@ import java.net.URL;
import java.net.URLEncoder;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
+import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -235,6 +237,7 @@ public class KMSClientProvider extends K
private SSLFactory sslFactory;
private ConnectionConfigurator configurator;
private DelegationTokenAuthenticatedURL.Token authToken;
+ private UserGroupInformation loginUgi;
@Override
public String toString() {
@@ -316,6 +319,7 @@ public class KMSClientProvider extends K
KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
new EncryptedQueueRefiller());
authToken = new DelegationTokenAuthenticatedURL.Token();
+loginUgi = UserGroupInformation.getCurrentUser();
}
private String createServiceURL(URL url) throws IOException {
@@ -374,14 +378,29 @@ public class KMSClientProvider extends K
return conn;
}
- private HttpURLConnection createConnection(URL url, String method)
+ private HttpURLConnection createConnection(final URL url, String method)
throws IOException {
HttpURLConnection conn;
try {
- DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
- conn = authUrl.openConnection(url, authToken);
-} catch (AuthenticationException ex) {
+ // if current UGI is different from UGI at constructor time, behave as
+ // proxyuser
+ UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
+ final String doAsUser =
+ (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
+ ? null : currentUgi.getShortUserName();
+
+ // creating the HTTP connection using the current UGI at constructor time
+ conn = loginUgi.doAs(new PrivilegedExceptionAction() {
+@Override
+public HttpURLConnection run() throws Exception {
+ DelegationTokenAuthenticatedURL auth
svn commit: r1619551 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Author: tucu
Date: Thu Aug 21 19:00:03 2014
New Revision: 1619551
URL: http://svn.apache.org/r1619551
Log:
HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey performance.
(hitliuyi via tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619551&r1=1619550&r2=1619551&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 19:00:03 2014
@@ -300,6 +300,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10862. Miscellaneous trivial corrections to KMS classes.
(asuresh via tucu)
+HADOOP-10967. Improve DefaultCryptoExtension#generateEncryptedKey
+performance. (hitliuyi via tucu)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1619551&r1=1619550&r2=1619551&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Thu Aug 21 19:00:03 2014
@@ -219,6 +219,13 @@ public class KeyProviderCryptoExtension
private static class DefaultCryptoExtension implements CryptoExtension {
private final KeyProvider keyProvider;
+private static final ThreadLocal RANDOM =
+new ThreadLocal() {
+ @Override
+ protected SecureRandom initialValue() {
+return new SecureRandom();
+ }
+};
private DefaultCryptoExtension(KeyProvider keyProvider) {
this.keyProvider = keyProvider;
@@ -233,10 +240,10 @@ public class KeyProviderCryptoExtension
"No KeyVersion exists for key '%s' ", encryptionKeyName);
// Generate random bytes for new key and IV
Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
- SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
final byte[] newKey = new byte[encryptionKey.getMaterial().length];
- random.nextBytes(newKey);
- final byte[] iv = random.generateSeed(cipher.getBlockSize());
+ RANDOM.get().nextBytes(newKey);
+ final byte[] iv = new byte[cipher.getBlockSize()];
+ RANDOM.get().nextBytes(iv);
// Encryption key IV is derived from new key's IV
final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
// Encrypt the new key
svn commit: r1619545 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/ hadoop-common/src/test/java/org/apache/hadoop
Author: tucu
Date: Thu Aug 21 18:59:48 2014
New Revision: 1619545
URL: http://svn.apache.org/r1619545
Log:
HADOOP-10936. Change default KeyProvider bitlength to 128. (wang)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619545&r1=1619544&r2=1619545&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:48 2014
@@ -168,6 +168,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10793. KeyShell args should use single-dash style. (wang)
+HADOOP-10936. Change default KeyProvider bitlength to 128. (wang)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1619545&r1=1619544&r2=1619545&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
Thu Aug 21 18:59:48 2014
@@ -54,7 +54,7 @@ public abstract class KeyProvider {
public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding";
public static final String DEFAULT_BITLENGTH_NAME =
"hadoop.security.key.default.bitlength";
- public static final int DEFAULT_BITLENGTH = 256;
+ public static final int DEFAULT_BITLENGTH = 128;
/**
* The combination of both the key version name and the key material.
@@ -341,6 +341,16 @@ public abstract class KeyProvider {
public Map getAttributes() {
return (attributes == null) ? Collections.EMPTY_MAP : attributes;
}
+
+@Override
+public String toString() {
+ return "Options{" +
+ "cipher='" + cipher + '\'' +
+ ", bitLength=" + bitLength +
+ ", description='" + description + '\'' +
+ ", attributes=" + attributes +
+ '}';
+}
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1619545&r1=1619544&r2=1619545&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
Thu Aug 21 18:59:48 2014
@@ -445,7 +445,7 @@ public class KeyShell extends Configured
"by the argument within the provider specified by the\n" +
"-provider argument. You may specify a cipher with the -cipher\n" +
"argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" +
- "The default keysize is 256. You may specify the requested key\n" +
+ "The default keysize is 128. You may specify the requested key\n" +
"length using the -size argument. Arbitrary attribute=value\n" +
"style attributes may be specified using the -attr argument.\n" +
"-attr may be specified multiple times, once per attribute.\n";
@@ -479,7 +479,8 @@ public class KeyShell extends Configured
warnIfTransientProvider();
try {
provider.createKey(keyName, options
svn commit: r1619546 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/http/ hadoop-common/src/main/java/org/apache/hadoop/jmx/
Author: tucu
Date: Thu Aug 21 18:59:51 2014
New Revision: 1619546
URL: http://svn.apache.org/r1619546
Log:
HADOOP-10918. JMXJsonServlet fails when used within Tomcat. (tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619546&r1=1619545&r2=1619546&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:51 2014
@@ -287,6 +287,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10937. Need to set version name correctly before decrypting EEK.
(Arun Suresh via wang)
+HADOOP-10918. JMXJsonServlet fails when used within Tomcat. (tucu)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java?rev=1619546&r1=1619545&r2=1619546&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
Thu Aug 21 18:59:51 2014
@@ -1037,7 +1037,7 @@ public final class HttpServer2 implement
String remoteUser = request.getRemoteUser();
if (remoteUser == null) {
- response.sendError(HttpServletResponse.SC_UNAUTHORIZED,
+ response.sendError(HttpServletResponse.SC_FORBIDDEN,
"Unauthenticated users are not " +
"authorized to access this page.");
return false;
@@ -1045,7 +1045,7 @@ public final class HttpServer2 implement
if (servletContext.getAttribute(ADMINS_ACL) != null &&
!userHasAdministratorAccess(servletContext, remoteUser)) {
- response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "User "
+ response.sendError(HttpServletResponse.SC_FORBIDDEN, "User "
+ remoteUser + " is unauthorized to access this page.");
return false;
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1619546&r1=1619545&r2=1619546&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
Thu Aug 21 18:59:51 2014
@@ -140,6 +140,12 @@ public class JMXJsonServlet extends Http
mBeanServer = ManagementFactory.getPlatformMBeanServer();
}
+ protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
+ HttpServletResponse response) throws IOException {
+return HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
+request, response);
+ }
+
/**
* Process a GET request for the specified resource.
*
@@ -153,8 +159,7 @@ public class JMXJsonServlet extends Http
String jsonpcb = null;
PrintWriter writer = null;
try {
- if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
- request, response)) {
+ if (!isInstrumentationAccessAllowed(request, response)) {
return;
}
Modified:
svn commit: r1619544 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/crypto/key/ src/main/java/org/apache/hadoop/crypto/key/kms/ src/test/
Author: tucu
Date: Thu Aug 21 18:59:46 2014
New Revision: 1619544
URL: http://svn.apache.org/r1619544
Log:
HADOOP-10937. Need to set version name correctly before decrypting EEK.
Contributed by Arun Suresh.
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619544&r1=1619543&r2=1619544&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:46 2014
@@ -282,6 +282,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10920. site plugin couldn't parse hadoop-kms index.apt.vm.
(Akira Ajisaka via wang)
+HADOOP-10937. Need to set version name correctly before decrypting EEK.
+(Arun Suresh via wang)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1619544&r1=1619543&r2=1619544&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Thu Aug 21 18:59:46 2014
@@ -21,11 +21,13 @@ package org.apache.hadoop.crypto.key;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
+
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import com.google.common.base.Preconditions;
+
import org.apache.hadoop.classification.InterfaceAudience;
/**
@@ -97,7 +99,7 @@ public class KeyProviderCryptoExtension
public static EncryptedKeyVersion createForDecryption(String
encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
- KeyVersion encryptedKeyVersion = new KeyVersion(null, null,
+ KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
return new EncryptedKeyVersion(null, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
@@ -258,6 +260,13 @@ public class KeyProviderCryptoExtension
keyProvider.getKeyVersion(encryptionKeyVersionName);
Preconditions.checkNotNull(encryptionKey,
"KeyVersion name '%s' does not exist", encryptionKeyVersionName);
+ Preconditions.checkArgument(
+ encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
+.equals(KeyProviderCryptoExtension.EEK),
+"encryptedKey version name must be '%s', is '%s'",
+KeyProviderCryptoExtension.EEK,
+encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
+);
final byte[] encryptionKeyMaterial = encryptionKey.getMaterial();
// Encryption key IV is determined from encrypted key's IV
final byte[] encryptionIV =
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1619544&r1=1619543&r2=1619544&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/
svn commit: r1619547 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Author: tucu
Date: Thu Aug 21 18:59:52 2014
New Revision: 1619547
URL: http://svn.apache.org/r1619547
Log:
HADOOP-10939. Fix TestKeyProviderFactory testcases to use default 128 bit
length keys. Contributed by Arun Suresh.
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619547&r1=1619546&r2=1619547&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:52 2014
@@ -289,6 +289,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10918. JMXJsonServlet fails when used within Tomcat. (tucu)
+HADOOP-10939. Fix TestKeyProviderFactory testcases to use default 128 bit
+length keys. (Arun Suresh via wang)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1619547&r1=1619546&r2=1619547&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Thu Aug 21 18:59:52 2014
@@ -91,9 +91,9 @@ public class TestKeyProviderFactory {
static void checkSpecificProvider(Configuration conf,
String ourUrl) throws Exception {
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
-byte[] key1 = new byte[32];
-byte[] key2 = new byte[32];
-byte[] key3 = new byte[32];
+byte[] key1 = new byte[16];
+byte[] key2 = new byte[16];
+byte[] key3 = new byte[16];
for(int i =0; i < key1.length; ++i) {
key1[i] = (byte) i;
key2[i] = (byte) (i * 2);
@@ -137,7 +137,7 @@ public class TestKeyProviderFactory {
KeyProvider.options(conf).setBitLength(8));
assertTrue("should throw", false);
} catch (IOException e) {
- assertEquals("Wrong key length. Required 8, but got 256",
e.getMessage());
+ assertEquals("Wrong key length. Required 8, but got 128",
e.getMessage());
}
provider.createKey("key4", new byte[]{1},
KeyProvider.options(conf).setBitLength(8));
@@ -153,7 +153,7 @@ public class TestKeyProviderFactory {
provider.rollNewVersion("key4", key1);
assertTrue("should throw", false);
} catch (IOException e) {
- assertEquals("Wrong key length. Required 8, but got 256",
e.getMessage());
+ assertEquals("Wrong key length. Required 8, but got 128",
e.getMessage());
}
try {
provider.rollNewVersion("no-such-key", key1);
@@ -219,7 +219,7 @@ public class TestKeyProviderFactory {
public void checkPermissionRetention(Configuration conf, String ourUrl, Path
path) throws Exception {
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
// let's add a new key and flush and check that permissions are still set
to 777
-byte[] key = new byte[32];
+byte[] key = new byte[16];
for(int i =0; i < key.length; ++i) {
key[i] = (byte) i;
}
@@ -252,7 +252,7 @@ public class TestKeyProviderFactory {
conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY,
"javakeystoreprovider.password");
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
- provider.createKey("key3", new byte[32], KeyProvider.options(conf));
+ provider.createKey("key3", new byte[16], KeyProvider.options(conf));
provider.flush();
} catch (Exception ex) {
Assert.fail("could not create keystore with password file");
svn commit: r1619518 [2/3] - in /hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-common/ hadoop-common/dev-support/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoo
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java?rev=1619518&view=auto
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java
(added)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java
Thu Aug 21 18:58:53 2014
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.crypto.key.KeyProvider;
+
+import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A KeyProvider proxy implementation providing a short lived
+ * cache for KeyVersions to avoid burst of requests to hit the
+ * underlying KeyProvider.
+ */
+public class KMSCacheKeyProvider extends KeyProvider {
+ private final KeyProvider provider;
+ private LoadingCache keyVersionCache;
+ private LoadingCache currentKeyCache;
+
+ private static class KeyNotFoundException extends Exception {
+private static final long serialVersionUID = 1L;
+ }
+
+ public KMSCacheKeyProvider(KeyProvider prov, long timeoutMillis) {
+this.provider = prov;
+keyVersionCache =
CacheBuilder.newBuilder().expireAfterAccess(timeoutMillis,
+TimeUnit.MILLISECONDS).build(new CacheLoader() {
+ @Override
+ public KeyVersion load(String key) throws Exception {
+KeyVersion kv = provider.getKeyVersion(key);
+if (kv == null) {
+ throw new KeyNotFoundException();
+}
+return kv;
+ }
+});
+// for current key we don't want to go stale for more than 1 sec
+currentKeyCache = CacheBuilder.newBuilder().expireAfterWrite(1000,
+TimeUnit.MILLISECONDS).build(new CacheLoader() {
+ @Override
+ public KeyVersion load(String key) throws Exception {
+KeyVersion kv = provider.getCurrentKey(key);
+if (kv == null) {
+ throw new KeyNotFoundException();
+}
+return kv;
+ }
+});
+ }
+
+ @Override
+ public KeyVersion getCurrentKey(String name) throws IOException {
+try {
+ return currentKeyCache.get(name);
+} catch (ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ if (cause instanceof KeyNotFoundException) {
+return null;
+ } else if (cause instanceof IOException) {
+throw (IOException) cause;
+ } else {
+throw new IOException(cause);
+ }
+}
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName)
+ throws IOException {
+try {
+ return keyVersionCache.get(versionName);
+} catch (ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ if (cause instanceof KeyNotFoundException) {
+return null;
+ } else if (cause instanceof IOException) {
+throw (IOException) cause;
+ } else {
+throw new IOException(cause);
+ }
+}
+ }
+
+ @Override
+ public List getKeys() throws IOException {
+return provider.getKeys();
+ }
+
+ @Override
+ public List getKeyVersions(String name)
+ throws IOException {
+return provider.getKeyVersions(name);
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+return provider.getMetadata(name);
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material,
+ Options options) throws IOException {
+return provider.createKey(name, material, options);
+ }
+
+ @Override
+ public
svn commit: r1619518 [1/3] - in /hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-common/ hadoop-common/dev-support/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoo
Author: tucu Date: Thu Aug 21 18:58:53 2014 New Revision: 1619518 URL: http://svn.apache.org/r1619518 Log: HADOOP-10433. Key Management Server based on KeyProvider API. (tucu) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt hadoop-project/pom.xml Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/dev-support/findbugsExcludeFile.xml hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/pom.xml hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSCacheKeyProvider.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSMDCFilter.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/libexec/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/sbin/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/tomcat/ROOT/WEB-INF/web.xml hadoop/common
svn commit: r1619543 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/CHANGES.txt hadoop-kms/src/site/apt/index.apt.vm
Author: tucu Date: Thu Aug 21 18:59:44 2014 New Revision: 1619543 URL: http://svn.apache.org/r1619543 Log: HADOOP-10920. site plugin couldn't parse hadoop-kms index.apt.vm. Contributed by Akira Ajisaka. Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619543&r1=1619542&r2=1619543&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 18:59:44 2014 @@ -279,6 +279,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is thread-unsafe. (benoyantony viat tucu) +HADOOP-10920. site plugin couldn't parse hadoop-kms index.apt.vm. +(Akira Ajisaka via wang) + Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm?rev=1619543&r1=1619542&r2=1619543&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm Thu Aug 21 18:59:44 2014 @@ -106,14 +106,14 @@ Hadoop Key Management Server (KMS) - Doc ** KMS Aggregated Audit logs -Audit logs are aggregated for API accesses to the GET_KEY_VERSION, -GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations. + Audit logs are aggregated for API accesses to the GET_KEY_VERSION, + GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations. -Entries are grouped by the (user,key,operation) combined key for a configurable -aggregation interval after which the number of accesses to the specified -end-point by the user for a given key is flushed to the audit log. + Entries are grouped by the (user,key,operation) combined key for a + configurable aggregation interval after which the number of accesses to the + specified end-point by the user for a given key is flushed to the audit log. -The Aggregation interval is configured via the property : + The Aggregation interval is configured via the property : +---+
svn commit: r1619537 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/ hadoop-common/src/main/java/org/apache/hadoop
Author: tucu
Date: Thu Aug 21 18:59:32 2014
New Revision: 1619537
URL: http://svn.apache.org/r1619537
Log:
HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey in
the REST API. (asuresh via tucu)
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619537&r1=1619536&r2=1619537&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:32 2014
@@ -154,6 +154,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10750. KMSKeyProviderCache should be in hadoop-common.
(asuresh via tucu)
+HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey
+in the REST API. (asuresh via tucu)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1619537&r1=1619536&r2=1619537&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Thu Aug 21 18:59:32 2014
@@ -27,17 +27,19 @@ import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
/**
* A KeyProvider with Cytographic Extensions specifically for generating
* Encrypted Keys as well as decrypting them
*
*/
+@InterfaceAudience.Private
public class KeyProviderCryptoExtension extends
KeyProviderExtension {
- protected static final String EEK = "EEK";
- protected static final String EK = "EK";
+ public static final String EEK = "EEK";
+ public static final String EK = "EK";
/**
* This is a holder class whose instance contains the keyVersionName, iv
@@ -82,6 +84,14 @@ public class KeyProviderCryptoExtension
public interface CryptoExtension extends KeyProviderExtension.Extension {
/**
+ * Calls to this method allows the underlying K
svn commit: r1619542 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/KeyShell.java src/test/java/org/apache/hadoop/cry
Author: tucu
Date: Thu Aug 21 18:59:43 2014
New Revision: 1619542
URL: http://svn.apache.org/r1619542
Log:
HADOOP-10793. KeyShell args should use single-dash style. (wang)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619542&r1=1619541&r2=1619542&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:43 2014
@@ -166,6 +166,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10756. KMS audit log should consolidate successful similar requests.
(asuresh via tucu)
+HADOOP-10793. KeyShell args should use single-dash style. (wang)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1619542&r1=1619541&r2=1619542&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
Thu Aug 21 18:59:43 2014
@@ -38,9 +38,9 @@ import org.apache.hadoop.util.ToolRunner
*/
public class KeyShell extends Configured implements Tool {
final static private String USAGE_PREFIX = "Usage: hadoop key " +
- "[generic options]\n";
+ "[generic options]\n";
final static private String COMMANDS =
- " [--help]\n" +
+ " [-help]\n" +
" [" + CreateCommand.USAGE + "]\n" +
" [" + RollCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" +
@@ -90,11 +90,11 @@ public class KeyShell extends Configured
/**
* Parse the command line arguments and initialize the data
*
- * % hadoop key create keyName [--size size] [--cipher algorithm]
- *[--provider providerPath]
- * % hadoop key roll keyName [--provider providerPath]
+ * % hadoop key create keyName [-size size] [-cipher algorithm]
+ *[-provider providerPath]
+ * % hadoop key roll keyName [-provider providerPath]
* % hadoop key list [-provider providerPath]
- * % hadoop key delete keyName [--provider providerPath] [-i]
+ * % hadoop key delete keyName [-provider providerPath] [-i]
*
* @param args Command line arguments.
* @return 0 on success, 1 on failure.
@@ -107,47 +107,47 @@ public class KeyShell extends Configured
for (int i = 0; i < args.length; i++) { // parse command line
boolean moreTokens = (i < args.length - 1);
if (args[i].equals("create")) {
-String keyName = "--help";
+String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new CreateCommand(keyName, options);
-if ("--help".equals(keyName)) {
+if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if (args[i].equals("delete")) {
-String keyName = "--help";
+String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new DeleteCommand(keyName);
-if ("--help".equals(keyName)) {
+if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if (args[i].equals("roll")) {
-String keyName = "--help";
+String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new RollCommand(keyName);
-if ("--help".equals(keyName)) {
+if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if ("list".equals(args[i])) {
svn commit: r1619524 - in /hadoop/common/branches/branch-2/hadoop-common-project: hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/crypto/key/ hadoop-common/src/main/java/org/apache/hadoop
Author: tucu
Date: Thu Aug 21 18:59:07 2014
New Revision: 1619524
URL: http://svn.apache.org/r1619524
Log:
HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
(tucu)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619524&r1=1619523&r2=1619524&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:07 2014
@@ -120,6 +120,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
+HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
+(tucu)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1619524&r1=1619523&r2=1619524&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
Thu Aug 21 18:59:07 2014
@@ -270,7 +270,7 @@ public class JavaKeyStoreProvider extend
e);
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
- options.getDescription(), new Date(), 1);
+ options.getDescription(), options.getAttributes(), new Date(), 1);
if (options.getBitLength() != 8 * material.length) {
throw new IOException("Wrong key length. Required " +
options.getBitLength() + ", but got " + (8 * material.length));
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1619524&r1=1619523&r2=1619524&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
Thu Aug 21 18:59:07 2014
@@ -26,8 +26,11 @@ import java.io.OutputStreamWriter;
import java.net.URI;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
+import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
@@ -107,18 +110,22 @@ public abstr
svn commit: r1619534 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/crypto/key/ src/test/java/org/apache/hadoop/crypto/key/
Author: tucu
Date: Thu Aug 21 18:59:24 2014
New Revision: 1619534
URL: http://svn.apache.org/r1619534
Log:
HADOOP-10841. EncryptedKeyVersion should have a key name property. (asuresh via
tucu)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619534&r1=1619533&r2=1619534&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:24 2014
@@ -145,6 +145,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10824. Refactor KMSACLs to avoid locking. (Benoy Antony via
umamahesh)
+HADOOP-10841. EncryptedKeyVersion should have a key name property.
+(asuresh via tucu)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1619534&r1=1619533&r2=1619534&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Thu Aug 21 18:59:24 2014
@@ -44,17 +44,23 @@ public class KeyProviderCryptoExtension
* used to generate the encrypted Key and the encrypted KeyVersion
*/
public static class EncryptedKeyVersion {
+private String keyName;
private String keyVersionName;
private byte[] iv;
private KeyVersion encryptedKey;
-protected EncryptedKeyVersion(String keyVersionName, byte[] iv,
-KeyVersion encryptedKey) {
+protected EncryptedKeyVersion(String keyName, String keyVersionName,
+byte[] iv, KeyVersion encryptedKey) {
+ this.keyName = keyName;
this.keyVersionName = keyVersionName;
this.iv = iv;
this.encryptedKey = encryptedKey;
}
+public String getKeyName() {
+ return keyName;
+}
+
public String getKeyVersionName() {
return keyVersionName;
}
@@ -153,7 +159,8 @@ public class KeyProviderCryptoExtension
cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(keyVer.getMaterial(),
"AES"), new IvParameterSpec(flipIV(iv)));
byte[] ek = cipher.doFinal(newKey);
- return new EncryptedKeyVersion(keyVersion.getVersionName(), iv,
+ return new EncryptedKeyVersion(keyVersion.getName(),
+ keyVersion.getVersionName(), iv,
new KeyVersion(keyVer.getName(), EEK, ek));
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java?rev=1619534&r1=1619533&r2=1619534&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
Thu Aug 21 18:59:24 2014
@@ -45,6 +45,7 @@ public class TestKeyProviderCryptoExtens
kpExt.generateEncryptedKey(kv);
Assert.assertEquals(KeyProviderCryptoExtension.EEK,
ek1.getEncryptedKey().getVersionName());
+Assert.assertEquals("foo", ek1.getKeyName());
Assert.assertNotNull(ek1.getEncryptedKey().getMaterial());
Assert.assertEquals(kv.getMaterial().length,
ek1.getEncryptedKey().getMaterial().length);
svn commit: r1619529 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
Author: tucu
Date: Thu Aug 21 18:59:16 2014
New Revision: 1619529
URL: http://svn.apache.org/r1619529
Log:
HADOOP-10812. Delegate KeyProviderExtension#toString to underlying KeyProvider.
(wang)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619529&r1=1619528&r2=1619529&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:16 2014
@@ -135,6 +135,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10769. Create KeyProvider extension to handle delegation tokens.
(Arun Suresh via atm)
+HADOOP-10812. Delegate KeyProviderExtension#toString to underlying
+KeyProvider. (wang)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java?rev=1619529&r1=1619528&r2=1619529&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
Thu Aug 21 18:59:16 2014
@@ -120,4 +120,9 @@ public abstract class KeyProviderExtensi
public void flush() throws IOException {
keyProvider.flush();
}
+
+ @Override
+ public String toString() {
+return getClass().getSimpleName() + ": " + keyProvider.toString();
+ }
}
svn commit: r1619533 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tucu Date: Thu Aug 21 18:59:22 2014 New Revision: 1619533 URL: http://svn.apache.org/r1619533 Log: HADOOP-10817. ProxyUsers configuration should support configurable prefixes. (tucu) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619533&r1=1619532&r2=1619533&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 18:59:22 2014 @@ -119,6 +119,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey methods to KeyProvider. (asuresh via tucu) +HADOOP-10817. ProxyUsers configuration should support configurable + prefixes. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
svn commit: r1619538 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
Author: tucu
Date: Thu Aug 21 18:59:34 2014
New Revision: 1619538
URL: http://svn.apache.org/r1619538
Log:
HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is thread-unsafe.
(benoyantony viat tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619538&r1=1619537&r2=1619538&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:34 2014
@@ -265,6 +265,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10816. KeyShell returns -1 on error to the shell, should be 1.
(Mike Yoder via wang)
+HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is
+thread-unsafe. (benoyantony viat tucu)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java?rev=1619538&r1=1619537&r2=1619538&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
Thu Aug 21 18:59:34 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.ServiceLoader;
@@ -47,6 +48,15 @@ public abstract class KeyProviderFactory
private static final ServiceLoader serviceLoader =
ServiceLoader.load(KeyProviderFactory.class);
+ // Iterate through the serviceLoader to avoid lazy loading.
+ // Lazy loading would require synchronization in concurrent use cases.
+ static {
+Iterator iterServices = serviceLoader.iterator();
+while (iterServices.hasNext()) {
+ iterServices.next();
+}
+ }
+
public static List getProviders(Configuration conf
) throws IOException {
List result = new ArrayList();
svn commit: r1619540 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Author: tucu
Date: Thu Aug 21 18:59:38 2014
New Revision: 1619540
URL: http://svn.apache.org/r1619540
Log:
HADOOP-10891. Add EncryptedKeyVersion factory method to
KeyProviderCryptoExtension. (wang)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619540&r1=1619539&r2=1619540&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 21 18:59:38 2014
@@ -160,6 +160,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey
in the REST API. (asuresh via tucu)
+HADOOP-10891. Add EncryptedKeyVersion factory method to
+KeyProviderCryptoExtension. (wang)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1619540&r1=1619539&r2=1619540&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
Thu Aug 21 18:59:38 2014
@@ -80,6 +80,30 @@ public class KeyProviderCryptoExtension
}
/**
+ * Factory method to create a new EncryptedKeyVersion that can then be
+ * passed into {@link #decryptEncryptedKey}. Note that the fields of the
+ * returned EncryptedKeyVersion will only partially be populated; it is not
+ * necessarily suitable for operations besides decryption.
+ *
+ * @param encryptionKeyVersionName Version name of the encryption key used
+ * to encrypt the encrypted key.
+ * @param encryptedKeyIv Initialization vector of the encrypted
+ * key. The IV of the encryption key used
to
+ * encrypt the encrypted key is derived
from
+ * this IV.
+ * @param encryptedKeyMaterial Key material of the encrypted key.
+ * @return EncryptedKeyVersion suitable for decryption.
+ */
+public static EncryptedKeyVersion createForDecryption(String
+encryptionKeyVersionName, byte[] encryptedKeyIv,
+byte[] encryptedKeyMaterial) {
+ KeyVersion encryptedKeyVersion = new KeyVersion(null, null,
+ encryptedKeyMaterial);
+ return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ encryptedKeyIv, encryptedKeyVersion);
+}
+
+/**
* @return Name of the encryption key used to encrypt the encrypted key.
*/
public String getEncryptionKeyName() {
