SLIDER-546 Slider conf dir files are not being picked up by the AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a97af55a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a97af55a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a97af55a Branch: refs/heads/develop Commit: a97af55accce5dd59fa6b1f0dfa99f4c69073762 Parents: 7e4abb1 Author: Steve Loughran <ste...@apache.org> Authored: Tue Oct 21 12:25:44 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Oct 21 12:25:44 2014 +0100 ---------------------------------------------------------------------- slider-assembly/src/main/scripts/slider | 2 +- slider-assembly/src/main/scripts/slider.py | 3 +-- .../org/apache/slider/client/SliderClient.java | 12 ++++----- .../apache/slider/common/tools/SliderUtils.java | 2 +- .../core/launch/ClasspathConstructor.java | 2 +- .../server/appmaster/SliderAppMaster.java | 27 ++++++++++++++------ 6 files changed, 29 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-assembly/src/main/scripts/slider ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider index 24ae64e..787016a 100755 --- a/slider-assembly/src/main/scripts/slider +++ b/slider-assembly/src/main/scripts/slider @@ -55,7 +55,7 @@ if [ -f "${confdir}/slider-env.sh" ]; then . "${confdir}/slider-env.sh" fi -slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}" +slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=${confdir}" slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts} # allow for an extra classpath http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-assembly/src/main/scripts/slider.py ---------------------------------------------------------------------- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index 2ee172e..94e40a2 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -33,7 +33,7 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" HADOOP_CONF_DIR = "HADOOP_CONF_DIR" SLIDER_CLASSNAME = "org.apache.slider.Slider" -DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s" +DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=%s" ON_POSIX = 'posix' in sys.builtin_module_names @@ -211,7 +211,6 @@ def java(classname, args, classpath, jvm_opts_list): """ # split the JVM opts by space # java = "/usr/bin/java" - prg = "java" if os.environ["JAVA_HOME"] is not None and os.environ["JAVA_HOME"]: prg = os.path.join(os.environ["JAVA_HOME"], "bin", "java") else: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index dc0cab0..5482566 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1146,7 +1146,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); - // conf directory setup + + // look for the configuration directory named on the command line Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = @@ -1160,11 +1161,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); - log.debug("Copying AM configuration data from {}", localConfDirPath); - remoteConfPath = new Path(clusterDirectory, - SliderKeys.SUBMITTED_CONF_DIR); - SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, - null); + log.debug("Slide configuration directory is {}; remote to be {}", + localConfDirPath, SliderKeys.SUBMITTED_CONF_DIR); + remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); + SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index c034e98..a6d8cd5 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1210,10 +1210,10 @@ public final class SliderUtils { // and nothing else classpath.appendAll(classpath.localJVMClasspath()); } else { - classpath.addLibDir(libdir); if (sliderConfDir != null) { classpath.addClassDirectory(sliderConfDir); } + classpath.addLibDir(libdir); classpath.addRemoteClasspathEnvVar(); classpath.appendAll(classpath.yarnApplicationClasspath(config)); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java index 8e49435..f474ffd 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java @@ -113,7 +113,7 @@ public class ClasspathConstructor { } public void addClassDirectory(String pathToDir) { - append(buildLibDir(appendDirectoryTerminator(pathToDir))); + append(appendDirectoryTerminator(pathToDir)); } public void insertClassDirectory(String pathToDir) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index a73fef1..78584c3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -23,8 +23,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.DataOutputBuffer; @@ -78,6 +80,7 @@ import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.SliderAMArgs; import org.apache.slider.common.params.SliderAMCreateAction; @@ -375,6 +378,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ public SliderAppMaster() { super(SERVICE_CLASSNAME_SHORT); + new HdfsConfiguration(); + new YarnConfiguration(); } /* =================================================================== */ @@ -383,25 +388,31 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @Override //AbstractService public synchronized void serviceInit(Configuration conf) throws Exception { - + // slider client if found + + Configuration customConf = SliderUtils.loadClientConfigurationResource(); // Load in the server configuration - if it is actually on the Classpath Configuration serverConf = ConfigHelper.loadFromResource(SERVER_RESOURCE); - ConfigHelper.mergeConfigurations(conf, serverConf, SERVER_RESOURCE, true); + ConfigHelper.mergeConfigurations(customConf, serverConf, SERVER_RESOURCE, true); + serviceArgs.applyDefinitions(customConf); + serviceArgs.applyFileSystemBinding(customConf); + // conf now contains all customizations AbstractActionArgs action = serviceArgs.getCoreAction(); SliderAMCreateAction createAction = (SliderAMCreateAction) action; - //sort out the location of the AM - serviceArgs.applyDefinitions(conf); - serviceArgs.applyFileSystemBinding(conf); + // sort out the location of the AM String rmAddress = createAction.getRmAddress(); if (rmAddress != null) { log.debug("Setting rm address from the command line: {}", rmAddress); - SliderUtils.setRmSchedulerAddress(conf, rmAddress); + SliderUtils.setRmSchedulerAddress(customConf, rmAddress); } - serviceArgs.applyDefinitions(conf); - serviceArgs.applyFileSystemBinding(conf); + + log.info("AM configuration:\n{}", + ConfigHelper.dumpConfigToString(customConf)); + + ConfigHelper.mergeConfigurations(conf, customConf, CLIENT_RESOURCE, true); //init security with our conf if (SliderUtils.isHadoopClusterSecure(conf)) { log.info("Secure mode with kerberos realm {}",
