SLIDER-540 Command Test Base to set up HADOOP_CONF_DIR
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1ce0716e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1ce0716e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1ce0716e Branch: refs/heads/develop Commit: 1ce0716e1badb4957ef63afdc434d7702723b9d1 Parents: 25e00d9 Author: Steve Loughran <ste...@apache.org> Authored: Mon Oct 20 12:37:12 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Oct 20 12:37:12 2014 +0100 ---------------------------------------------------------------------- .../slider/common/tools/ConfigHelper.java | 39 ++++++++++-- .../abstracttests/AbstractTestBuildSetup.groovy | 1 - .../funtest/framework/CommandTestBase.groovy | 63 +++++++++++++++---- .../slider/funtest/framework/ConfLoader.groovy | 55 ++++++++++++++++- .../funtest/framework/FuntestProperties.groovy | 9 ++- .../slider/funtest/framework/SliderShell.groovy | 65 ++++++++++++++++---- 6 files changed, 198 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index b710224..c1b7e89 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -81,7 +81,7 @@ public class ConfigHelper { * @return the sorted keyset */ - public static TreeSet<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { + public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { TreeSet<String> sorted = new TreeSet<String>(); for (Map.Entry<String, String> entry : conf) { sorted.add(entry.getKey()); @@ -339,7 +339,7 @@ public class ConfigHelper { try { conf.addResource(file.toURI().toURL()); } catch (MalformedURLException e) { - //should never happen... + // should never happen... throw new IOException( "File " + file.toURI() + " doesn't have a valid URL"); } @@ -347,6 +347,33 @@ public class ConfigHelper { } /** + * Add a configuration from a file to an existing configuration + * @param conf existing configuration + * @param file file to load + * @return the merged configuration + * @throws IOException + */ + public static Configuration addConfigurationFile(Configuration conf, + File file) + throws IOException { + Configuration c2 = loadConfFromFile(file, false); + mergeConfigurations(conf, c2, file.getAbsolutePath()); + return conf; + } + + /** + * Add the system env variables with the given prefix (by convention, env.) + * @param conf existing configuration + * @param prefix prefix + */ + public static void addEnvironmentVariables(Configuration conf, String prefix) { + Map<String, String> env = System.getenv(); + for (Map.Entry<String, String> entry : env.entrySet()) { + conf.set(prefix + entry.getKey(),entry.getValue(), "env"); + } + } + + /** * looks for the config under $confdir/$templateFilename; if not there * loads it from /conf/templateFile. * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the @@ -384,8 +411,8 @@ public class ConfigHelper { */ public static Configuration loadTemplateConfiguration(FileSystem fs, Path templatePath, - String fallbackResource) throws - IOException { + String fallbackResource) + throws IOException { Configuration conf; String origin; if (fs.exists(templatePath)) { @@ -490,8 +517,8 @@ public class ConfigHelper { * @return the loaded configuration * @throws FileNotFoundException if the resource is missing */ - public static Configuration loadMandatoryResource(String resource) throws - FileNotFoundException { + public static Configuration loadMandatoryResource(String resource) + throws FileNotFoundException { Configuration conf = new Configuration(false); URL resURL = ConfigHelper.class.getClassLoader() .getResource(resource); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy index 104de58..9f0c4d7 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy @@ -137,7 +137,6 @@ abstract class AbstractTestBuildSetup extends SliderTestUtils implements Funtest def fs = HadoopFS.get(path.toUri(), conf) assert fs.exists(path) - } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 744ec66..efe3f60 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.registry.client.api.RegistryConstants import org.apache.hadoop.util.ExitUtil -import org.apache.hadoop.util.Shell import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.common.tools.ConfigHelper import org.apache.slider.core.main.LauncherExitCodes @@ -81,10 +80,13 @@ abstract class CommandTestBase extends SliderTestUtils { public static final String TEST_AM_KEYTAB static File keytabFile - + /* + Static initializer for test configurations. If this code throws exceptions + (which it may) the class will not be instantiable. + */ static { ConfigHelper.registerDeprecatedConfigItems(); - SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML); + SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true); THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, KEY_TEST_THAW_WAIT_TIME, 1000 * DEFAULT_THAW_WAIT_TIME_SECONDS) @@ -111,6 +113,21 @@ abstract class CommandTestBase extends SliderTestUtils { @BeforeClass public static void setupTestBase() { Configuration conf = loadSliderConf(); + + SliderShell.confDir = SLIDER_CONF_DIRECTORY + SliderShell.scriptFile = SliderShell.windows ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT + + //set the property of the configuration directory + def path = SLIDER_CONF_DIRECTORY.absolutePath + SLIDER_CONFIG.set(ENV_SLIDER_CONF_DIR, path) + // locate any hadoop conf dir + def hadoopConf = SLIDER_CONFIG.getTrimmed(ENV_HADOOP_CONF_DIR) + if (hadoopConf) { + File hadoopConfDir = new File(hadoopConf).canonicalFile + // propagate the value to the client config + SliderShell.setEnv(ENV_HADOOP_CONF_DIR, hadoopConfDir.absolutePath) + } + if (SliderUtils.maybeInitSecurity(conf)) { log.debug("Security enabled") SliderUtils.forceLogin() @@ -129,16 +146,13 @@ abstract class CommandTestBase extends SliderTestUtils { } else { log.info "Security is off" } - SliderShell.confDir = SLIDER_CONF_DIRECTORY - SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") - } /** - * give our thread a name + * give the test thread a name */ @Before public void nameThread() { @@ -146,8 +160,28 @@ abstract class CommandTestBase extends SliderTestUtils { } /** - * Add a jar to the slider classpath - * @param clazz + * Add a configuration file at a given path + * @param dir directory containing the file + * @param filename filename + * @return true if the file was found + * @throws IOException loading problems (other than a missing file) + */ + public static boolean maybeAddConfFile(File dir, String filename) throws IOException { + File confFile = new File(dir, filename) + if (confFile.isFile()) { + ConfigHelper.addConfigurationFile(SLIDER_CONFIG, confFile) + log.debug("Loaded $confFile") + return true; + } else { + log.debug("Did not find $confFile âskipping load") + return false; + } + } + + /** + * Add a jar to the slider classpath by looking up a class and determining + * its containing JAR + * @param clazz class inside the JAR */ public static void addExtraJar(Class clazz) { def jar = SliderUtils.findContainingJarOrFail(clazz) @@ -158,6 +192,12 @@ abstract class CommandTestBase extends SliderTestUtils { } } + /** + * Resolve a system property, throwing an exception if it is not present + * @param key property name + * @return the value + * @throws RuntimeException if the property is not set + */ public static String sysprop(String key) { def property = System.getProperty(key) if (!property) { @@ -173,8 +213,9 @@ abstract class CommandTestBase extends SliderTestUtils { static void println(String s) { System.out.println(s) } + /** - * Print to system out + * Print a newline to system out * @param string */ static void println() { @@ -208,7 +249,7 @@ abstract class CommandTestBase extends SliderTestUtils { * @return */ public static Configuration loadSliderConf() { - Configuration conf = ConfLoader.loadSliderConf(SLIDER_CONF_XML) + Configuration conf = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true) return conf } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy index 447ea87..8cefb84 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy @@ -18,17 +18,66 @@ package org.apache.slider.funtest.framework +import groovy.util.logging.Slf4j +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.common.tools.ConfigHelper +import static org.apache.slider.funtest.framework.FuntestProperties.* + +@Slf4j public class ConfLoader { - public static YarnConfiguration loadSliderConf(File confFile) { + public static YarnConfiguration loadSliderConf(File confFile, boolean loadHadoopConfDir = true) { URI confURI = confFile.toURI(); YarnConfiguration conf = new YarnConfiguration() def confXmlUrl = confURI.toURL() conf.addResource(confXmlUrl) - conf.set(FuntestProperties.KEY_TEST_CONF_XML, confXmlUrl.toString()) - conf.set(FuntestProperties.KEY_TEST_CONF_DIR, CommandTestBase.SLIDER_CONF_DIRECTORY.absolutePath) + conf.set(KEY_TEST_CONF_XML, confXmlUrl.toString()) + + def sliderConfDir = confFile.parent + conf.set(KEY_TEST_CONF_DIR, sliderConfDir) + conf.set(ENV_SLIDER_CONF_DIR, sliderConfDir) + ConfigHelper.addEnvironmentVariables(conf, ENV_PREFIX) + + if (loadHadoopConfDir) { + def hadoopConf = conf.getTrimmed(ENV_HADOOP_CONF_DIR) + if (hadoopConf) { + File hadoopConfDir = new File(hadoopConf).canonicalFile + def hadoopConfDirPath = hadoopConfDir.absolutePath + if (!hadoopConfDir.directory) { + throw new FileNotFoundException(hadoopConfDirPath) + } + log.debug("$ENV_HADOOP_CONF_DIR=$hadoopConfDirPath âloading") + // a conf dir has been set. Load the values locally + maybeAddConfFile(conf, hadoopConfDir, CORE_SITE_XML) + maybeAddConfFile(conf, hadoopConfDir, HDFS_SITE_XML) + maybeAddConfFile(conf, hadoopConfDir, YARN_SITE_XML) + } + + } + return conf } + + /** + * Add a configuration file at a given path + * @param dir directory containing the file + * @param filename filename + * @return true if the file was found + * @throws IOException loading problems (other than a missing file) + */ + public static boolean maybeAddConfFile(Configuration conf, File dir, String filename) + throws IOException { + File confFile = new File(dir, filename) + if (confFile.isFile()) { + ConfigHelper.addConfigurationFile(conf, confFile) + log.info("Loaded ${confFile.absolutePath}") + return true; + } else { + log.info("Did not find ${confFile.absolutePath} âskipping load") + return false; + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy index d9d6dd6..7d38ffc 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy @@ -50,9 +50,10 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { String CLIENT_CONFIG_FILENAME = SliderKeys.CLIENT_RESOURCE - String ENV_CONF_DIR = "SLIDER_CONF_DIR" + String ENV_SLIDER_CONF_DIR = "SLIDER_CONF_DIR" + String ENV_HADOOP_CONF_DIR = "HADOOP_CONF_DIR" String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" - + String SCRIPT_NAME = "slider" String KEY_TEST_CONF_XML = "slider.test.conf.xml" String KEY_TEST_CONF_DIR = "slider.test.conf.dir" @@ -69,4 +70,8 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { "slider.test.instance.launch.wait.seconds"; int DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS = 60 * 3; + String ENV_PREFIX = "env." + String CORE_SITE_XML = "core-site.xml" + String HDFS_SITE_XML = "hdfs-site.xml" + String YARN_SITE_XML = "yarn-site.xml" } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ce0716e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index a7fd58c..86595dc 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -42,6 +42,11 @@ class SliderShell extends Shell { public static final List<String> slider_classpath_extra = [] + /** + * Environment varaibles + */ + protected static final Map<String, String> environment = [:] + final String command /** @@ -49,7 +54,7 @@ class SliderShell extends Shell { * @param commands */ SliderShell(Collection<String> commands) { - super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH) + super(org.apache.hadoop.util.Shell.WINDOWS ? CMD : BASH) assert confDir != null; assert scriptFile != null; command = scriptFile.absolutePath + " " + commands.join(" ") @@ -60,17 +65,16 @@ class SliderShell extends Shell { * @return the script exit code */ int execute() { - String confDirCmd = env(FuntestProperties.ENV_CONF_DIR, confDir) log.info(command) - List<String> commandLine = [ - confDirCmd, - ] + setEnv(FuntestProperties.ENV_SLIDER_CONF_DIR, confDir) if (!slider_classpath_extra.empty) { - commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA, - SliderUtils.join(slider_classpath_extra, - (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"), - false)) + setEnv(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA, + SliderUtils.join(slider_classpath_extra, + pathElementSeparator, + false)) } + List<String> commandLine = buildEnvCommands() + commandLine << command String script = commandLine.join("\n") log.debug(script) @@ -79,14 +83,53 @@ class SliderShell extends Shell { return ret; } + public String getPathElementSeparator() { + File.pathSeparator + } + + public static boolean isWindows() { + return org.apache.hadoop.util.Shell.WINDOWS + } + + /** + * Set an environment variable + * @param var variable name + * @param val value + */ + public static void setEnv(String var, Object val) { + environment[var] = val.toString() + } + + /** + * Get an environment variable + * @param var variable name + * @return the value or null + */ + public static String getEnv(String var) { + return environment[var] + } + + /** + * Build up a list of environment variable setters from the + * env variables + * @return a list of commands to set up the env on the target system. + */ + public static List<String> buildEnvCommands() { + List<String> commands = [] + environment.each { String var, String val -> + commands << env(var, val) + } + return commands + } + /** * Add an environment variable * @param var variable * @param val value (which will be stringified) * @return an env variable command */ - String env(String var, Object val) { - if (org.apache.hadoop.util.Shell.WINDOWS) { + static String env(String var, Object val) { + if (isWindows()) { return "set " + var + "=${val.toString()}" } else { return "export " + var + "=${val.toString()};"
