SLIDER-171 changes to setting up java command and classpath needed to work with cross-platform client
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/853a9790 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/853a9790 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/853a9790 Branch: refs/heads/develop Commit: 853a979006d5a08734a63ff80ce9c15ab577d61e Parents: abb3d01 Author: Steve Loughran <ste...@apache.org> Authored: Tue Jul 1 21:09:51 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Jul 1 21:09:51 2014 +0100 ---------------------------------------------------------------------- .../apache/slider/common/tools/SliderUtils.java | 2 +- .../core/launch/ClasspathConstructor.java | 4 +-- .../core/launch/JavaCommandLineBuilder.java | 2 +- .../funtest/framework/FileUploader.groovy | 30 ++++++++++++++------ .../lifecycle/TestAgentClusterLifecycle.groovy | 3 +- 5 files changed, 28 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/853a9790/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 5479b54..62e61c9 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1146,7 +1146,7 @@ public final class SliderUtils { // and nothing else classpath.appendAll(classpath.javaVMClasspath()); } else { - classpath.addLibDir("./" + libdir); + classpath.addLibDir(libdir); if (sliderConfDir != null) { classpath.addClassDirectory(sliderConfDir); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/853a9790/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java index ee4c4d1..ca325d1 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java @@ -38,8 +38,8 @@ import java.util.List; */ public class ClasspathConstructor { - // public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR; - public static final String CLASS_PATH_SEPARATOR = File.pathSeparator; + public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR; +// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator; private final List<String> pathElements = new ArrayList<>(); public ClasspathConstructor() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/853a9790/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java index e8b6eb1..7b60461 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java @@ -38,7 +38,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder { * @return the path to the Java binary */ protected String getJavaBinary() { - return ApplicationConstants.Environment.JAVA_HOME.$() + "/bin/java"; + return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"; } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/853a9790/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index 2dc85be..921adbf 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -18,6 +18,7 @@ package org.apache.slider.funtest.framework +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS @@ -29,6 +30,7 @@ import org.apache.hadoop.security.UserGroupInformation @SuppressWarnings("GroovyOctalInteger") @Slf4j +@CompileStatic class FileUploader { final Configuration conf final UserGroupInformation user @@ -69,8 +71,8 @@ class FileUploader { return FileUtil.copy(src, fs, destPath, false, conf) } catch (AccessControlException ace) { log.error("No write access to destination directory $dir" + - "Ensure home directory exists and has correct permissions." + - ace, ace) + "Ensure home directory exists and has correct permissions. $ace", + ace) throw ace } } else { @@ -103,9 +105,10 @@ class FileUploader { } } - public def getFileSystemAsUserHdfs() { - def hdfs = UserGroupInformation.createRemoteUser("hdfs") - getFileSystem(hdfs, HadoopFS.getDefaultUri(conf)) + public def getFileSystemAsUserName(String username) { + + def user = UserGroupInformation.createRemoteUser(username) + getFileSystem(user, HadoopFS.getDefaultUri(conf)) } /** @@ -127,11 +130,22 @@ class FileUploader { throw ace; } //now create as hdfs - def FsAsUserHDFS = fileSystemAsUserHdfs - FsAsUserHDFS.mkdirs(home, new FsPermission((short) 00755)) - FsAsUserHDFS.setOwner(home, user.userName, user.primaryGroupName) + try { + attemptToCreateHomeDir("hdfs", home) + } catch (AccessControlException ace2) { + + log.info("Failed to mkdir $home as $user -impersonating 'hadoop'") + attemptToCreateHomeDir("hadoop", home) + + } } } return home } + + public void attemptToCreateHomeDir(String username, Path home) { + def privilegedFS = getFileSystemAsUserName(username) + privilegedFS.mkdirs(home, new FsPermission((short) 00755)) + privilegedFS.setOwner(home, user.userName, user.primaryGroupName) + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/853a9790/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy index 06809cc..25a4134 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy @@ -36,7 +36,7 @@ import org.junit.Test @CompileStatic @Slf4j public class TestAgentClusterLifecycle extends AgentCommandTestBase -implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { static String CLUSTER = "test_agent_cluster_lifecycle" @@ -47,6 +47,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { @Before public void prepareCluster() { setupCluster(CLUSTER) + describe("Create a 0-role cluster, so testing AM start/stop") } @After