Repository: hadoop Updated Branches: refs/heads/branch-2 c47d72d8d -> 8bf2a0de6
HADOOP-11070. Create MiniKMS for testing. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e98c2447 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e98c2447 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e98c2447 Branch: refs/heads/branch-2 Commit: e98c244730337477c0fe7c19c984ee4581ff567f Parents: c47d72d Author: Alejandro Abdelnur <t...@apache.org> Authored: Fri Sep 5 14:09:22 2014 -0700 Committer: Alejandro Abdelnur <t...@apache.org> Committed: Fri Sep 5 22:01:06 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++++++++++++++++++ .../hadoop/crypto/key/kms/server/TestKMS.java | 82 +------- 4 files changed, 211 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 724cfac..492d41a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -168,6 +168,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11060. Create a CryptoCodec test that verifies interoperability between the JCE and OpenSSL implementations. (hitliuyi via tucu) + HADOOP-11070. Create MiniKMS for testing. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 527454b..481f80e 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -222,9 +222,9 @@ </goals> <configuration> <target> - <mkdir dir="${project.build.directory}/test-classes/webapp"/> + <mkdir dir="${project.build.directory}/test-classes/kms-webapp"/> - <copy todir="${project.build.directory}/test-classes/webapp"> + <copy todir="${project.build.directory}/test-classes/kms-webapp"> <fileset dir="${basedir}/src/main/webapp"/> </copy> </target> http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java new file mode 100644 index 0000000..5a6d4c5 --- /dev/null +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key.kms.server; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.security.SslSocketConnector; +import org.mortbay.jetty.webapp.WebAppContext; + +import java.io.File; +import java.io.FileWriter; +import java.io.Writer; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.ServerSocket; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +public class MiniKMS { + + private static Server createJettyServer(String keyStore, String password) { + try { + boolean ssl = keyStore != null; + InetAddress localhost = InetAddress.getByName("localhost"); + String host = "localhost"; + ServerSocket ss = new ServerSocket(0, 50, localhost); + int port = ss.getLocalPort(); + ss.close(); + Server server = new Server(0); + if (!ssl) { + server.getConnectors()[0].setHost(host); + server.getConnectors()[0].setPort(port); + } else { + SslSocketConnector c = new SslSocketConnector(); + c.setHost(host); + c.setPort(port); + c.setNeedClientAuth(false); + c.setKeystore(keyStore); + c.setKeystoreType("jks"); + c.setKeyPassword(password); + server.setConnectors(new Connector[]{c}); + } + return server; + } catch (Exception ex) { + throw new RuntimeException("Could not start embedded servlet container, " + + ex.getMessage(), ex); + } + } + + private static URL getJettyURL(Server server) { + boolean ssl = server.getConnectors()[0].getClass() + == SslSocketConnector.class; + try { + String scheme = (ssl) ? "https" : "http"; + return new URL(scheme + "://" + + server.getConnectors()[0].getHost() + ":" + + server.getConnectors()[0].getPort()); + } catch (MalformedURLException ex) { + throw new RuntimeException("It should never happen, " + ex.getMessage(), + ex); + } + } + + public static class Builder { + private File kmsConfDir; + private String log4jConfFile; + private File keyStoreFile; + private String keyStorePassword; + + public Builder() { + kmsConfDir = new File("target/test-classes").getAbsoluteFile(); + log4jConfFile = "kms-log4j.properties"; + } + + public Builder setKmsConfDir(File confDir) { + Preconditions.checkNotNull(confDir, "KMS conf dir is NULL"); + Preconditions.checkArgument(confDir.exists(), + "KMS conf dir does not exist"); + kmsConfDir = confDir; + return this; + } + + public Builder setLog4jConfFile(String log4jConfFile) { + Preconditions.checkNotNull(log4jConfFile, "log4jconf file is NULL"); + this.log4jConfFile = log4jConfFile; + return this; + } + + public Builder setSslConf(File keyStoreFile, String keyStorePassword) { + Preconditions.checkNotNull(keyStoreFile, "keystore file is NULL"); + Preconditions.checkNotNull(keyStorePassword, "keystore password is NULL"); + Preconditions.checkArgument(keyStoreFile.exists(), + "keystore file does not exist"); + this.keyStoreFile = keyStoreFile; + this.keyStorePassword = keyStorePassword; + return this; + } + + public MiniKMS build() { + Preconditions.checkArgument(kmsConfDir.exists(), + "KMS conf dir does not exist"); + return new MiniKMS(kmsConfDir.getAbsolutePath(), log4jConfFile, + (keyStoreFile != null) ? keyStoreFile.getAbsolutePath() : null, + keyStorePassword); + } + } + + private String kmsConfDir; + private String log4jConfFile; + private String keyStore; + private String keyStorePassword; + private Server jetty; + private URL kmsURL; + + public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore, + String password) { + this.kmsConfDir = kmsConfDir; + this.log4jConfFile = log4ConfFile; + this.keyStore = keyStore; + this.keyStorePassword = password; + } + + public void start() throws Exception { + System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir); + File aclsFile = new File(kmsConfDir, "kms-acls.xml"); + if (!aclsFile.exists()) { + Configuration acls = new Configuration(false); + Writer writer = new FileWriter(aclsFile); + acls.writeXml(writer); + writer.close(); + } + File coreFile = new File(kmsConfDir, "core-site.xml"); + if (!coreFile.exists()) { + Configuration core = new Configuration(); + Writer writer = new FileWriter(coreFile); + core.writeXml(writer); + writer.close(); + } + File kmsFile = new File(kmsConfDir, "kms-site.xml"); + if (!kmsFile.exists()) { + Configuration kms = new Configuration(false); + kms.set("hadoop.security.key.provider.path", + "jceks://file@" + kmsConfDir + "/kms.keystore"); + kms.set("hadoop.kms.authentication.type", "simple"); + Writer writer = new FileWriter(kmsFile); + kms.writeXml(writer); + writer.close(); + } + System.setProperty("log4j.configuration", log4jConfFile); + jetty = createJettyServer(keyStore, keyStorePassword); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource("kms-webapp"); + if (url == null) { + throw new RuntimeException( + "Could not find kms-webapp/ dir in test classpath"); + } + WebAppContext context = new WebAppContext(url.getPath(), "/kms"); + jetty.addHandler(context); + jetty.start(); + kmsURL = new URL(getJettyURL(jetty), "kms"); + } + + public URL getKMSUrl() { + return kmsURL; + } + + public void stop() { + if (jetty != null && jetty.isRunning()) { + try { + jetty.stop(); + jetty = null; + } catch (Exception ex) { + throw new RuntimeException("Could not stop MiniKMS embedded Jetty, " + + ex.getMessage(), ex); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 52f6354..f381fa0 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -36,10 +36,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.webapp.WebAppContext; import javax.security.auth.Subject; import javax.security.auth.kerberos.KerberosPrincipal; @@ -52,7 +48,6 @@ import java.io.IOException; import java.io.Writer; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.SocketTimeoutException; import java.net.URI; @@ -91,49 +86,6 @@ public class TestKMS { return file; } - public static Server createJettyServer(String keyStore, String password) { - try { - boolean ssl = keyStore != null; - InetAddress localhost = InetAddress.getByName("localhost"); - String host = "localhost"; - ServerSocket ss = new ServerSocket(0, 50, localhost); - int port = ss.getLocalPort(); - ss.close(); - Server server = new Server(0); - if (!ssl) { - server.getConnectors()[0].setHost(host); - server.getConnectors()[0].setPort(port); - } else { - SslSocketConnector c = new SslSocketConnector(); - c.setHost(host); - c.setPort(port); - c.setNeedClientAuth(false); - c.setKeystore(keyStore); - c.setKeystoreType("jks"); - c.setKeyPassword(password); - server.setConnectors(new Connector[]{c}); - } - return server; - } catch (Exception ex) { - throw new RuntimeException("Could not start embedded servlet container, " - + ex.getMessage(), ex); - } - } - - public static URL getJettyURL(Server server) { - boolean ssl = server.getConnectors()[0].getClass() - == SslSocketConnector.class; - try { - String scheme = (ssl) ? "https" : "http"; - return new URL(scheme + "://" + - server.getConnectors()[0].getHost() + ":" + - server.getConnectors()[0].getPort()); - } catch (MalformedURLException ex) { - throw new RuntimeException("It should never happen, " + ex.getMessage(), - ex); - } - } - public static abstract class KMSCallable implements Callable<Void> { private URL kmsUrl; @@ -144,33 +96,19 @@ public class TestKMS { protected void runServer(String keystore, String password, File confDir, KMSCallable callable) throws Exception { - System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, - confDir.getAbsolutePath()); - System.setProperty("log4j.configuration", "log4j.properties"); - Server jetty = createJettyServer(keystore, password); + MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder().setKmsConfDir(confDir) + .setLog4jConfFile("log4j.properties"); + if (keystore != null) { + miniKMSBuilder.setSslConf(new File(keystore), password); + } + MiniKMS miniKMS = miniKMSBuilder.build(); + miniKMS.start(); try { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource("webapp"); - if (url == null) { - throw new RuntimeException( - "Could not find webapp/ dir in test classpath"); - } - WebAppContext context = new WebAppContext(url.getPath(), "/kms"); - jetty.addHandler(context); - jetty.start(); - url = new URL(getJettyURL(jetty), "kms"); - System.out.println("Test KMS running at: " + url); - callable.kmsUrl = url; + System.out.println("Test KMS running at: " + miniKMS.getKMSUrl()); + callable.kmsUrl = miniKMS.getKMSUrl(); callable.call(); } finally { - if (jetty != null && jetty.isRunning()) { - try { - jetty.stop(); - } catch (Exception ex) { - throw new RuntimeException("Could not stop embedded Jetty, " + - ex.getMessage(), ex); - } - } + miniKMS.stop(); } }
