SLIDER-254 bump Accumulo version to 1.6.1-SNAPSHOT and configure app package to use CredentialProvider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e0e7261 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e0e7261 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e0e7261 Branch: refs/heads/develop Commit: 6e0e72613085fc242f1602ca455a30403f267511 Parents: 02e9f83 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Mon Aug 4 15:11:15 2014 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Mon Aug 4 15:11:15 2014 -0700 ---------------------------------------------------------------------- app-packages/accumulo/appConfig.json | 6 ++-- .../slider/accumulo/CustomAuthenticator.java | 31 +++++++++++--------- .../funtest/accumulo/AccumuloBasicIT.groovy | 23 +++++++++------ pom.xml | 2 +- .../org/apache/slider/client/SliderClient.java | 12 ++------ 5 files changed, 37 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json index 83d35c7..c024c97 100644 --- a/app-packages/accumulo/appConfig.json +++ b/app-packages/accumulo/appConfig.json @@ -27,13 +27,11 @@ "site.global.monitor_protocol": "http", "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data", "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}", - "site.accumulo-site.instance.secret": "DEFAULT", "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator", - "site.accumulo-site.instance.security.credential.provider": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", + "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", "site.accumulo-site.tserver.memory.maps.max": "80M", "site.accumulo-site.tserver.cache.data.size": "7M", "site.accumulo-site.tserver.cache.index.size": "20M", - "site.accumulo-site.trace.token.property.password": "secret", "site.accumulo-site.trace.user": "root", "site.accumulo-site.tserver.sort.buffer.size": "50M", "site.accumulo-site.tserver.walog.max.size": "100M", @@ -46,7 +44,7 @@ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar," }, "credentials": { - "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root", "instance.secret"] + "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"] }, "components": { "ACCUMULO_MASTER": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java index 45df8cb..dd18a8e 100644 --- a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java +++ b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java @@ -38,8 +38,8 @@ import java.util.List; import java.util.Set; public final class CustomAuthenticator implements Authenticator { - private static final String CREDENTIAL_PROVIDER_PROPERTY = - "instance.security.credential.provider"; + public static final String ROOT_INITIAL_PASSWORD_PROPERTY = + "root.initial.password"; private static ZKAuthenticator zkAuthenticator = null; public CustomAuthenticator() { @@ -55,14 +55,15 @@ public final class CustomAuthenticator implements Authenticator { public void initializeSecurity(TCredentials credentials, String principal, byte[] token) throws AccumuloSecurityException { char[] pass = null; - SiteConfiguration siteconf = SiteConfiguration.getInstance - (DefaultConfiguration.getInstance()); - String jksFile = siteconf.getAllPropertiesWithPrefix( - Property.INSTANCE_PREFIX).get(CREDENTIAL_PROVIDER_PROPERTY); + SiteConfiguration siteconf = SiteConfiguration.getInstance( + DefaultConfiguration.getInstance()); + String jksFile = siteconf.get( + Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS); if (jksFile == null) { - throw new RuntimeException("instance.security.credential.provider not " + - "specified in accumulo-site.xml"); + throw new RuntimeException( + Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS + + " not specified in accumulo-site.xml"); } try { Configuration conf = new Configuration(); @@ -73,25 +74,27 @@ public final class CustomAuthenticator implements Authenticator { if (providers != null) { for (CredentialProvider provider : providers) { try { - CredentialEntry entry = provider.getCredentialEntry(principal); + CredentialEntry entry = provider.getCredentialEntry( + ROOT_INITIAL_PASSWORD_PROPERTY); if (entry != null) { pass = entry.getCredential(); break; } } catch (IOException ioe) { - throw new IOException("Can't get key " + principal + " from " + + throw new IOException("Can't get key " + + ROOT_INITIAL_PASSWORD_PROPERTY + " from " + provider.getClass().getName() + ", " + jksFile, ioe); } } } } catch (IOException ioe) { - throw new RuntimeException("Can't get key " + principal + " from " + - jksFile, ioe); + throw new RuntimeException("Can't get key " + + ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile, ioe); } if (pass == null) { - throw new RuntimeException("Can't get key " + principal + " from " + - jksFile); + throw new RuntimeException("Can't get key " + + ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile); } zkAuthenticator.initializeSecurity(credentials, principal, new String(pass).getBytes(StandardCharsets.UTF_8)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index b8e0841..63bb091 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy @@ -17,12 +17,14 @@ package org.apache.slider.funtest.accumulo import groovy.util.logging.Slf4j +import org.apache.accumulo.core.conf.Property import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.security.ProviderUtils import org.apache.hadoop.security.UserGroupInformation import org.apache.hadoop.security.alias.CredentialProvider import org.apache.hadoop.security.alias.CredentialProviderFactory +import org.apache.slider.accumulo.CustomAuthenticator import org.apache.slider.api.ClusterDescription import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderKeys @@ -38,8 +40,8 @@ import org.junit.Test @Slf4j class AccumuloBasicIT extends AccumuloAgentCommandTestBase { - protected static final String PROVIDER = - "site.accumulo-site.instance.security.credential.provider" + protected static final String PROVIDER_PROPERTY = "site.accumulo-site." + + Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS protected ConfTree tree @Before @@ -50,12 +52,11 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase { "skipping creation of credentials" SliderClient.replaceTokens(tree, UserGroupInformation.getCurrentUser() .getShortUserName(), getClusterName()) - String jks = tree.global.get(PROVIDER) + String jks = tree.global.get(PROVIDER_PROPERTY) def keys = tree.credentials.get(jks) - assert keys!=null, "jks specified in $PROVIDER wasn't requested in " + - "credentials" - assert keys.size()==2, "test expects root and instance.secret to be " + - "requested" + assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " + + "in credentials" + assert keys.size()==3, "test expects 3 passwords to be requested" Path jksPath = ProviderUtils.unnestUri(new URI(jks)) if (clusterFS.exists(jksPath)) { clusterFS.delete(jksPath, false) @@ -64,8 +65,12 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase { conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, jks) CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0) - provider.createCredentialEntry(USER, PASSWORD.toCharArray()) - provider.createCredentialEntry("instance.secret", INSTANCE_SECRET.toCharArray()) + provider.createCredentialEntry( + CustomAuthenticator.ROOT_INITIAL_PASSWORD_PROPERTY, PASSWORD.toCharArray()) + provider.createCredentialEntry(Property.INSTANCE_SECRET.toString(), + INSTANCE_SECRET.toCharArray()) + provider.createCredentialEntry(Property.TRACE_TOKEN_PROPERTY_PREFIX + .toString() + "password", PASSWORD.toCharArray()) provider.flush() assert clusterFS.exists(jksPath), "jks $jks not created" log.info("Created credential provider $jks for test") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7e2063c..ab6e9a6 100644 --- a/pom.xml +++ b/pom.xml @@ -120,7 +120,7 @@ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version> <hbase.version>0.98.4-hadoop2</hbase.version> - <accumulo.version>1.6.0</accumulo.version> + <accumulo.version>1.6.1-SNAPSHOT</accumulo.version> <!-- artifact versions http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e0e7261/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ed5b74c..99acfbc 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -29,7 +29,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.CredentialShell; -import org.apache.hadoop.security.alias.CredentialShell.PasswordReader; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -110,27 +109,20 @@ import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.curator.CuratorServiceInstance; import org.apache.slider.server.services.registry.SliderRegistryService; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; - -import static org.apache.slider.common.params.SliderActions.*; - import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.Console; import java.io.File; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; -import java.io.InputStreamReader; import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -141,6 +133,8 @@ import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; +import static org.apache.slider.common.params.SliderActions.*; + /** * Client service for Slider */ @@ -497,7 +491,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe continue; } String[] csarg = new String[]{ - "create", alias, "--provider", provider}; + "create", alias, "-provider", provider}; log.info("Creating credentials for {} in {}", alias, provider); credentialShell.run(csarg); }
