[01/47] git commit: SLIDER-386 fixed role-specific failure threshold
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 6e883bf91 -> 5784c4ab5 SLIDER-386 fixed role-specific failure threshold Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ecab238 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ecab238 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ecab238 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 7ecab238a919ae4013f23b259ab4eba317635485 Parents: 82cf1f0 Author: Billie Rinaldi Authored: Tue Sep 2 08:48:10 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 2 08:48:10 2014 -0700 -- .../java/org/apache/slider/server/appmaster/state/AppState.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ecab238/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 07976ef..8b70185 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1559,7 +1559,7 @@ public class AppState { ConfTreeOperations resources = instanceDefinition.getResourceOperations(); return resources.getComponentOptInt(roleStatus.getName(), -ResourceKeys.CONTAINER_FAILURE_SHORTLIFE, +ResourceKeys.CONTAINER_FAILURE_THRESHOLD, failureThreshold); }
[15/47] git commit: SLIDER-350 use gawk as awk implementation for windows; add checks for that and xargs on the command line
SLIDER-350 use gawk as awk implementation for windows; add checks for that and xargs on the command line Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/690793a4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/690793a4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/690793a4 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 690793a4f8a82a3b7395271803ae9af89eb2cdda Parents: 2e3c4c8 Author: Steve Loughran Authored: Thu Sep 4 14:00:01 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 14:00:01 2014 +0100 -- .../common/tools/TestWindowsSupport.groovy | 38 ++-- .../apache/slider/test/SliderTestUtils.groovy | 2 +- 2 files changed, 37 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy index 84356ea..d18fe76 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy @@ -105,12 +105,33 @@ class TestWindowsSupport extends SliderTestBase { } + + @Test + public void testHasGawkInstalled() throws Throwable { +assume(Shell.WINDOWS, "not windows") +assert 0 == exec(["gawk", "--version"]) + } + + @Test + public void testHasXargsInstalled() throws Throwable { +assume(Shell.WINDOWS, "not windows") +assert 0 == exec(["xargs", "--version"]) + } + + @Test public void testEmitKillCommand() throws Throwable { killJavaProcesses("regionserver", 9) } @Test + public void testHadoopHomeDefined() throws Throwable { +assume(Shell.WINDOWS, "not windows") +def hadoopHome = Shell.hadoopHome +log.info("HADOOP_HOME=$hadoopHome") + } + + @Test public void testHasWinutils() throws Throwable { assume(Shell.WINDOWS, "not windows") SliderUtils.verifyWinUtilsValid() @@ -123,14 +144,27 @@ class TestWindowsSupport extends SliderTestBase { assert winUtilsPath File winUtils = new File(winUtilsPath) log.debug("Winutils is at $winUtils)") + +ForkedProcessService process = exec([winUtilsPath, "systeminfo"]) +assert 0 == process.exitCode + } + + + /** + * Exec a set of commands, wait a few seconds for it to finish. + * @param commands + * @return + */ + public ForkedProcessService exec(ArrayList commands) { ForkedProcessService process; process = new ForkedProcessService( -methodName.methodName, +methodName.methodName, [:], -[winUtilsPath, "systeminfo"]); +commands); process.init(new Configuration()); EndOfServiceWaiter waiter = new EndOfServiceWaiter(process); process.start(); waiter.waitForServiceToStop(5000); +process } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 58a76af..89f1a34 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -755,7 +755,7 @@ class SliderTestUtils extends Assert { /* "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID" */ - GString killCommand = "\"jps -l | grep \"${grepString}\" | awk \"{print \$1}\" | xargs -n 1 taskkill /PID\"" + GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /PID\"" commandString = ["CMD", "/C", killCommand] } Process command = commandString.execute()
[26/47] git commit: Merge branch 'feature/SLIDER-394_env_check' into develop
Merge branch 'feature/SLIDER-394_env_check' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88d72ddf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88d72ddf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88d72ddf Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 88d72ddfb32977ff098b0329ae6f3faf2f7c2875 Parents: 6320f61 2d6f276 Author: Steve Loughran Authored: Thu Sep 4 17:12:02 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 17:12:02 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 18 +- .../common/params/AbstractActionArgs.java | 11 + .../slider/common/params/ActionHelpArgs.java| 14 +- .../slider/common/params/ActionVersionArgs.java | 12 +- .../apache/slider/common/tools/SliderUtils.java | 557 +-- .../slider/core/build/InstanceBuilder.java | 5 +- .../server/appmaster/SliderAppMaster.java | 5 +- .../services/utility/EndOfServiceWaiter.java| 84 +++ .../services/workflow/ForkedProcessService.java | 21 +- .../services/workflow/LongLivedProcess.java | 57 +- .../agent/AgentMiniClusterTestBase.groovy | 9 + .../tools/TestExecutionEnvironment.groovy | 42 ++ .../common/tools/TestWindowsSupport.groovy | 4 +- .../slider/providers/agent/AgentTestBase.groovy | 10 + .../services/workflow/EndOfServiceWaiter.java | 56 -- .../TestWorkflowForkedProcessService.java | 4 +- .../masterless/TestRoleOptPropagation.groovy| 4 +- 17 files changed, 663 insertions(+), 250 deletions(-) --
[14/47] git commit: SLIDER-350 add strict test for windows utils being valid
SLIDER-350 add strict test for windows utils being valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2e3c4c81 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2e3c4c81 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2e3c4c81 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 2e3c4c81b799cbf6d06e394c2b17f8f161f68734 Parents: bfdec58 Author: Steve Loughran Authored: Thu Sep 4 13:46:59 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 13:46:59 2014 +0100 -- .../apache/slider/common/tools/SliderUtils.java | 50 +++- .../common/tools/TestWindowsSupport.groovy | 30 ++-- 2 files changed, 76 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java -- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index a864878..eb214db 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -37,6 +37,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; @@ -63,6 +64,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; @@ -104,8 +106,9 @@ public final class SliderUtils { public static final String JAVA_SECURITY_KRB5_REALM = "java.security.krb5.realm"; public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc"; + public static final String WINUTILS = "WINUTILS.EXE"; + - private SliderUtils() { } @@ -1499,4 +1502,49 @@ public final class SliderUtils { return is; } + + /** + * Strictly verify that windows utils is present. + * Checks go as far as opening the file and looking for + * the headers. + * @throws IOException on any problem reading the file + * @throws FileNotFoundException if the file is not considered valid + */ + public static void verifyWinUtilsValid() throws IOException { +if (!Shell.WINDOWS) { + return; +} +String winUtilsPath = Shell.getWinUtilsPath(); +if (winUtilsPath == null) { + throw new FileNotFoundException(WINUTILS + " not found on Path : " + + System.getenv("Path")); +} +File winUtils = new File(winUtilsPath); +if (!winUtils.isFile()) { + throw new FileNotFoundException(WINUTILS + + " at " + winUtilsPath + + " is not a file"); + +} +if (winUtils.length() < 0x100) { + throw new FileNotFoundException(WINUTILS ++ " at " + winUtilsPath ++ " is too short to be an executable"); +} +// now read two bytes and verify the header. +FileReader reader = null; +try { + int[] header = new int[2]; + reader = new FileReader(winUtilsPath); + header[0] = reader.read(); + header[1] = reader.read(); + if (header[0] != 'M' || header[1] != 'Z') { +throw new FileNotFoundException(WINUTILS + + " at " + winUtilsPath + + " is not a windows executable file"); + } +} finally { + IOUtils.closeStream(reader); +} + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy index 7f5edf3..84356ea 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy @@ -27,14 +27,16 @@ import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.util.Shell import org.apache.slider.providers.agent.AgentUtils -import o
[02/47] git commit: SLIDER-387 adjusted default accumulo configuration
SLIDER-387 adjusted default accumulo configuration Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ac2536f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ac2536f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ac2536f Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 0ac2536f96e9010569deb0a0d2079f8c23c17e06 Parents: 7ecab23 Author: Billie Rinaldi Authored: Tue Sep 2 11:17:46 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 2 11:17:46 2014 -0700 -- app-packages/accumulo/appConfig.json| 5 +++-- .../accumulo/src/test/resources/appConfig_monitor_ssl.json | 5 +++-- app-packages/accumulo/src/test/resources/appConfig_ssl.json | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/appConfig.json -- diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json index a44dbad..2c0c2d5 100644 --- a/app-packages/accumulo/appConfig.json +++ b/app-packages/accumulo/appConfig.json @@ -9,7 +9,7 @@ "site.global.app_log_dir": "${AGENT_LOG_ROOT}", "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}", -"site.global.tserver_heapsize": "128m", +"site.global.tserver_heapsize": "256m", "site.global.master_heapsize": "128m", "site.global.monitor_heapsize": "64m", "site.global.gc_heapsize": "64m", @@ -27,12 +27,13 @@ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}", "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator", "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", +"site.accumulo-site.tserver.memory.maps.native.enabled": "false", "site.accumulo-site.tserver.memory.maps.max": "80M", "site.accumulo-site.tserver.cache.data.size": "7M", "site.accumulo-site.tserver.cache.index.size": "20M", "site.accumulo-site.trace.user": "root", "site.accumulo-site.tserver.sort.buffer.size": "50M", -"site.accumulo-site.tserver.walog.max.size": "100M", +"site.accumulo-site.tserver.walog.max.size": "40M", "site.accumulo-site.master.port.client": "0", "site.accumulo-site.trace.port.client": "0", "site.accumulo-site.tserver.port.client": "0", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json -- diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json index c6a2639..45e6e19 100644 --- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json +++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json @@ -9,7 +9,7 @@ "site.global.app_log_dir": "${AGENT_LOG_ROOT}", "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}", -"site.global.tserver_heapsize": "128m", +"site.global.tserver_heapsize": "256m", "site.global.master_heapsize": "128m", "site.global.monitor_heapsize": "64m", "site.global.gc_heapsize": "64m", @@ -27,12 +27,13 @@ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}", "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator", "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", +"site.accumulo-site.tserver.memory.maps.native.enabled": "false", "site.accumulo-site.tserver.memory.maps.max": "80M", "site.accumulo-site.tserver.cache.data.size": "7M", "site.accumulo-site.tserver.cache.index.size": "20M", "site.accumulo-site.trace.user": "root", "site.accumulo-site.tserver.sort.buffer.size": "50M", -"site.accumulo-site.tserver.walog.max.size": "100M", +"site.accumulo-site.tserver.walog.max.size": "40M", "site.accumulo-site.master.port.client": "0", "site.accumulo-site.trace.port.client": "0", "site.accumulo-site.tserver.port.client": "0", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_ssl.json -- diff --git a/app-packages/accumulo/src/test/resources/appC
[45/47] git commit: Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f14d32c6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f14d32c6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f14d32c6 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f14d32c6fac2e252e587dbdc88e0e3e079f44d37 Parents: 2d6331a b323df8 Author: Steve Loughran Authored: Fri Sep 5 13:59:58 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 5 13:59:58 2014 +0100 -- pom.xml | 1 + slider-agent/pom.xml| 4 +-- slider-agent/src/main/python/setup.py | 2 +- slider-agent/src/test/python/python-wrap| 40 -- slider-agent/src/test/python/python-wrap.sh | 42 slider-agent/src/test/python/unitTests.py | 36 ++-- slider-core/pom.xml | 4 +-- 7 files changed, 65 insertions(+), 64 deletions(-) --
[33/47] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/930920d2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/930920d2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/930920d2 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 930920d245bb354cbfbec43391a203bd6db04fa4 Parents: b7f37ea 6126db6 Author: Steve Loughran Authored: Thu Sep 4 19:00:55 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 19:00:55 2014 +0100 -- pom.xml | 41 +++-- 1 file changed, 39 insertions(+), 2 deletions(-) --
[37/47] git commit: Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e2a80326 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e2a80326 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e2a80326 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: e2a80326a7f7ddc6f2bc8b00dbaa5265726a6bb0 Parents: 930920d ee3ad77 Author: Steve Loughran Authored: Thu Sep 4 21:02:30 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 21:02:30 2014 +0100 -- .../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +- .../test/groovy/org/apache/slider/test/SliderTestUtils.groovy| 4 ++-- .../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) --
[44/47] git commit: SLIDER-401 agent tests on windows
SLIDER-401 agent tests on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b323df81 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b323df81 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b323df81 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b323df811a138a0880b60be95d9150419d1dcb85 Parents: ce3f700 Author: Steve Loughran Authored: Fri Sep 5 13:53:05 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 5 13:53:05 2014 +0100 -- slider-agent/src/test/python/unitTests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b323df81/slider-agent/src/test/python/unitTests.py -- diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py index bab121f..6aa0167 100644 --- a/slider-agent/src/test/python/unitTests.py +++ b/slider-agent/src/test/python/unitTests.py @@ -70,8 +70,9 @@ def all_tests_suite(): else: for file_name in files_list: if file_name.endswith(PY_EXT) and not file_name == __file__: -logger.info(file_name) -tests_list.append(file_name.replace(PY_EXT, '')) +replaced = file_name.replace(PY_EXT, '') +logger.info(replaced) +tests_list.append(replaced) logger.info('') suite = unittest.TestLoader().loadTestsFromNames(tests_list)
[19/47] git commit: Merge branch 'feature/SLIDER-377-windows' into develop
Merge branch 'feature/SLIDER-377-windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29a40f50 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29a40f50 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29a40f50 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 29a40f5094b4275fa4d7159d91f1c965cdc6da50 Parents: 38c816e 877a016 Author: Steve Loughran Authored: Thu Sep 4 14:41:54 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 14:41:54 2014 +0100 -- .../apache/slider/common/tools/SliderUtils.java | 50 - .../common/tools/TestWindowsSupport.groovy | 76 +++- .../apache/slider/test/SliderTestUtils.groovy | 2 +- 3 files changed, 123 insertions(+), 5 deletions(-) --
[27/47] git commit: SLIDER-377 tagging some test files as @CompileStatic
SLIDER-377 tagging some test files as @CompileStatic Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2965d8a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2965d8a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2965d8a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f2965d8a056fcf6d56a39f0841f1903048c95c29 Parents: 88d72dd Author: Steve Loughran Authored: Thu Sep 4 17:54:46 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 17:54:46 2014 +0100 -- .../slider/common/tools/TestClientResourceRegistration.groovy | 3 +++ .../org/apache/slider/common/tools/TestConfigHelperHDFS.groovy| 3 +++ .../org/apache/slider/common/tools/TestMiscSliderUtils.groovy | 3 +++ 3 files changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy index 7150b3c..e3bfa1b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy @@ -18,12 +18,15 @@ package org.apache.slider.common.tools +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.conf.YarnConfiguration import org.junit.Test @Slf4j +@CompileStatic + class TestClientResourceRegistration { /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy index 0d21d6c..d78aa1b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy @@ -18,6 +18,7 @@ package org.apache.slider.common.tools +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS @@ -27,6 +28,8 @@ import org.apache.slider.test.YarnMiniClusterTestBase import org.junit.Test @Slf4j +@CompileStatic + class TestConfigHelperHDFS extends YarnMiniClusterTestBase { //diabled for now; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy index 24367a3..9042af5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy @@ -18,12 +18,15 @@ package org.apache.slider.common.tools +import groovy.transform.CompileStatic import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.slider.test.SliderTestBase import org.junit.Test +@CompileStatic + class TestMiscSliderUtils extends SliderTestBase {
[10/47] git commit: SLIDER-181. Storm package updates
SLIDER-181. Storm package updates Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38c816eb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38c816eb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38c816eb Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 38c816ebbd3c883308bbe36fc00846d4ff4dac7b Parents: e9a353c Author: Sumit Mohanty Authored: Wed Sep 3 13:15:20 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 3 13:46:02 2014 -0700 -- app-packages/storm/README.txt | 19 ++--- app-packages/storm/appConfig.json | 10 +-- app-packages/storm/metainfo.xml | 2 +- ...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 app-packages/storm/pom.xml | 90 app-packages/storm/src/assembly/storm.xml | 68 +++ 6 files changed, 172 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/README.txt -- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index d4df85f..49adfce 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -17,19 +17,16 @@ How to create a Slider app package for Storm? -To create the app package you will need the Storm tarball copied to a specific location. -Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz. -So if you use a different version you may need to edit a few config values. +To create the app package you will need the Storm tarball and invoke mvn command +with appropriate parameters. -Replace the placeholder tarball for Storm. - cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/ - rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE +Command: +mvn clean package -Pstorm-app-package -Dpkg.version= + -Dpkg.name= -Dpkg.src= -Create a zip package at the root of the package (/app-packages/storm/) - zip -r apache-storm-0.9.3.zip . - -Verify the content using - unzip -l "$@" apache-storm-0.9.3.zip +Example: +mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578 + -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So its advisable that when you http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/appConfig.json -- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index b5fe436..df28e8f 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -3,13 +3,13 @@ "metadata": { }, "global": { -"application.def": "apache-storm-0.9.3.zip", +"application.def": "slider-storm-app-package-${pkg.version}.zip", "java_home": "/usr/jdk64/jdk1.7.0_45", "create.default.zookeeper.node": "true", "system_configs": "core-site", "site.global.app_user": "yarn", -"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578", +"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}", "site.global.user_group": "hadoop", "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_id": "Application2", @@ -22,8 +22,8 @@ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", -"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", -"site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", +"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireform
[03/47] git commit: SLIDER-388 allow default zookeeper port in ZookeeperUtils
SLIDER-388 allow default zookeeper port in ZookeeperUtils Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b525287d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b525287d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b525287d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b525287dd05a29af6308890aef44d524a6341296 Parents: 0ac2536 Author: Billie Rinaldi Authored: Tue Sep 2 12:39:06 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 2 12:39:06 2014 -0700 -- .../src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b525287d/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java -- diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java index 61b1ff0..cc1b2c9 100644 --- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java +++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; public class ZookeeperUtils { + public static final int DEFAULT_PORT = 2181; public static String buildConnectionString(String zkHosts, int port) { String zkPort = Integer.toString(port); @@ -73,7 +74,7 @@ public class ZookeeperUtils { List list = new ArrayList(len); if (strings != null) { for (String s : strings) { -list.add(HostAndPort.fromString(s.trim())); + list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT)); } } return list;
[17/47] git commit: SLIDER-350 windows support
SLIDER-350 windows support Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93c3c770 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93c3c770 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93c3c770 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 93c3c770fed878895e4fb67ac17cbbb098cb4fb3 Parents: 8f9caa9 Author: Steve Loughran Authored: Thu Sep 4 14:05:38 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 14:05:38 2014 +0100 -- .../common/tools/TestWindowsSupport.groovy | 22 +++- 1 file changed, 17 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/93c3c770/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy index 61ede62..6ae683a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy @@ -109,13 +109,13 @@ class TestWindowsSupport extends SliderTestBase { @Test public void testHasGawkInstalled() throws Throwable { assume(Shell.WINDOWS, "not windows") -assert 0 == exec(["gawk", "--version"]) +exec(0, ["gawk", "--version"]) } @Test public void testHasXargsInstalled() throws Throwable { assume(Shell.WINDOWS, "not windows") -assert 0 == exec(["xargs", "--version"]) +exec(0, ["xargs", "--version"]) } @@ -145,16 +145,28 @@ class TestWindowsSupport extends SliderTestBase { File winUtils = new File(winUtilsPath) log.debug("Winutils is at $winUtils)") -ForkedProcessService process = exec([winUtilsPath, "systeminfo"]) -assert 0 == process.exitCode +exec(0, [winUtilsPath, "systeminfo"]) } /** * Exec a set of commands, wait a few seconds for it to finish. + * @param status code * @param commands - * @return + * @return the process */ + public ForkedProcessService exec(int status, List commands) { +ForkedProcessService process = exec(commands) +assert status == process.exitCode +return process + } + + /** + * Exec a set of commands, wait a few seconds for it to finish. + * @param commands + * @return + */ + public ForkedProcessService exec(List commands) { ForkedProcessService process; process = new ForkedProcessService(
[43/47] git commit: SLIDER-401 agent tests on windows
SLIDER-401 agent tests on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce3f7009 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce3f7009 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce3f7009 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: ce3f7009e6e9fec204b264a9577a52632b9f3487 Parents: a9f0b98 Author: Steve Loughran Authored: Fri Sep 5 13:47:50 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 5 13:47:50 2014 +0100 -- slider-agent/src/main/python/setup.py | 2 +- slider-agent/src/test/python/unitTests.py | 31 -- 2 files changed, 15 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/main/python/setup.py -- diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py index 421b5f9..56969b6 100644 --- a/slider-agent/src/main/python/setup.py +++ b/slider-agent/src/main/python/setup.py @@ -17,7 +17,7 @@ from setuptools import setup setup( name = "slider-agent", -version = "0.31.0-incubating-SNAPSHOT", +version = "0.51.0-incubating-SNAPSHOT", packages = ['agent'], # metadata for upload to PyPI author = "Apache Software Foundation", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/test/python/unitTests.py -- diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py index b01438e..bab121f 100644 --- a/slider-agent/src/test/python/unitTests.py +++ b/slider-agent/src/test/python/unitTests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -''' +""" Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -16,14 +16,13 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -''' +""" import unittest -import doctest -from os.path import dirname, split, isdir -import logging.handlers +from os.path import isdir import logging -from random import shuffle +import os +import sys LOG_FILE_NAME='tests.log' SELECTED_PREFIX = "_" @@ -31,9 +30,9 @@ PY_EXT='.py' ignoredDirs = ["mock"] class TestAgent(unittest.TestSuite): - def run(self, result): + def run(self, result, debug=False): run = unittest.TestSuite.run -run(self, result) +run(self, result, debug) return result @@ -41,19 +40,19 @@ def parent_dir(path): if isdir(path): if path.endswith(os.sep): path = os.path.dirname(path) -parent_dir = os.path.dirname(path) +parent = os.path.dirname(path) else: -parent_dir = os.path.dirname(os.path.dirname(path)) +parent = os.path.dirname(os.path.dirname(path)) - return parent_dir + return parent def all_tests_suite(): - src_dir = os.getcwd() + root_dir = os.getcwd() files_list = [] - for directory in os.listdir(src_dir): + for directory in os.listdir(root_dir): if os.path.isdir(directory) and not directory in ignoredDirs: - files_list += os.listdir(src_dir + os.sep + directory) + files_list += os.listdir(root_dir + os.sep + directory) ## temporarily deleting to add more predictability ## shuffle(files_list) files_list.sort() @@ -99,9 +98,7 @@ def main(): logger.info('') if __name__ == '__main__': - import os - import sys - import io + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__ + os.sep + 'main' + os.sep + 'python') sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__ + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
[35/47] git commit: SLIDER-397 taskkill /f
SLIDER-397 taskkill /f Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27f0a625 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27f0a625 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27f0a625 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 27f0a625265de8e0d818917c94b2f0d5385b9044 Parents: 132d9f2 Author: Steve Loughran Authored: Thu Sep 4 20:39:05 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 20:39:05 2014 +0100 -- .../test/groovy/org/apache/slider/test/SliderTestUtils.groovy| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27f0a625/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 89f1a34..5086248 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -747,7 +747,7 @@ class SliderTestUtils extends Assert { def commandString if (!Shell.WINDOWS) { - GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal" + GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill /f $signal" log.info("Command command = $killCommand" ) commandString = ["bash", "-c", killCommand] @@ -755,7 +755,7 @@ class SliderTestUtils extends Assert { /* "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID" */ - GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /PID\"" + GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID " commandString = ["CMD", "/C", killCommand] } Process command = commandString.execute()
[20/47] git commit: SLIDER-146 added sample appConfig that enables kerberos for Accumulo
SLIDER-146 added sample appConfig that enables kerberos for Accumulo Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6320f618 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6320f618 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6320f618 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 6320f6186dec56f68afbbd638f5ddc0e7dc04bff Parents: 29a40f5 Author: Billie Rinaldi Authored: Thu Sep 4 07:15:22 2014 -0700 Committer: Billie Rinaldi Committed: Thu Sep 4 07:15:22 2014 -0700 -- .../src/test/resources/appConfig_kerberos.json | 55 1 file changed, 55 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6320f618/app-packages/accumulo/src/test/resources/appConfig_kerberos.json -- diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json new file mode 100644 index 000..c9684d4 --- /dev/null +++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json @@ -0,0 +1,55 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { +"application.def": "${app.package.name}.zip", +"java_home": "/usr/lib/jvm/java", +"site.global.app_user": "accumulo", +"site.global.app_log_dir": "${AGENT_LOG_ROOT}", +"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run", +"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}", +"site.global.tserver_heapsize": "256m", +"site.global.master_heapsize": "128m", +"site.global.monitor_heapsize": "64m", +"site.global.gc_heapsize": "64m", +"site.global.other_heapsize": "128m", +"site.global.hadoop_prefix": "/usr/lib/hadoop", +"site.global.hadoop_conf_dir": "/etc/hadoop/conf", +"site.global.zookeeper_home": "/usr/lib/zookeeper", +"site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}", +"site.global.accumulo_root_password": "NOT_USED", +"site.global.user_group": "accumulo", +"site.global.security_enabled": "false", +"site.global.ssl_cert_dir": "ssl", +"site.global.monitor_protocol": "http", +"site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data", +"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}", +"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator", +"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks", +"site.accumulo-site.general.kerberos.keytab": "/etc/security/keytabs/accumulo.service.keytab", +"site.accumulo-site.general.kerberos.principal": "accumulo/_h...@example.com", +"site.accumulo-site.tserver.memory.maps.native.enabled": "false", +"site.accumulo-site.tserver.memory.maps.max": "80M", +"site.accumulo-site.tserver.cache.data.size": "7M", +"site.accumulo-site.tserver.cache.index.size": "20M", +"site.accumulo-site.trace.user": "root", +"site.accumulo-site.tserver.sort.buffer.size": "50M", +"site.accumulo-site.tserver.walog.max.size": "40M", +"site.accumulo-site.master.port.client": "0", +"site.accumulo-site.trace.port.client": "0", +"site.accumulo-site.tserver.port.client": "0", +"site.accumulo-site.gc.port.client": "0", +"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}", +"site.accumulo-site.monitor.port.log4j": "0", +"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar," + }, + "credentials": { +"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"] + }, + "components": { +"slider-appmaster": { + "jvm.heapsize": "256M" +} + } +}
[11/47] git commit: Merge branch 'develop' into feature/auto_generate_storm_pkg
Merge branch 'develop' into feature/auto_generate_storm_pkg Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26b4153e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26b4153e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26b4153e Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 26b4153ebf070521fb5030d57846f4ccd7f45cb0 Parents: e1129b0 38c816e Author: Sumit Mohanty Authored: Wed Sep 3 13:46:44 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 3 13:46:44 2014 -0700 -- .../main/java/org/apache/slider/api/InternalKeys.java | 11 +++ .../main/java/org/apache/slider/api/ResourceKeys.java | 13 - .../apache/slider/server/appmaster/state/AppState.java | 9 + 3 files changed, 16 insertions(+), 17 deletions(-) --
[07/47] git commit: SLIDER-390 interpret short lived threshold as seconds instead of milliseconds
SLIDER-390 interpret short lived threshold as seconds instead of milliseconds Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0b7bd38 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0b7bd38 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0b7bd38 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b0b7bd38542ed287415066d0215c6fda8786336e Parents: 201686e Author: Billie Rinaldi Authored: Wed Sep 3 08:38:52 2014 -0700 Committer: Billie Rinaldi Committed: Wed Sep 3 08:38:52 2014 -0700 -- .../java/org/apache/slider/server/appmaster/state/AppState.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0b7bd38/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 8b70185..dce48d1 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1227,7 +1227,7 @@ public class AppState { /** * Is a role short lived by the threshold set for this application * @param instance instance - * @return true if the instance is considered short live + * @return true if the instance is considered short lived */ @VisibleForTesting public boolean isShortLived(RoleInstance instance) { @@ -1236,7 +1236,7 @@ public class AppState { boolean shortlived; if (started > 0) { long duration = time - started; - shortlived = duration < startTimeThreshold; + shortlived = duration < (startTimeThreshold * 1000); } else { // never even saw a start event shortlived = true;
git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows [created] be57d7a11 SLIDER-414 RAT is creating a dir that cannot be deleted Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1 Branch: refs/heads/feature/SLIDER-86-windows Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef Parents: 65d17d8 Author: Steve Loughran Authored: Tue Sep 9 18:26:52 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 18:26:52 2014 +0100 -- .../command-logger/application-pkg/pom.xml | 38 --- app-packages/command-logger/slider-pkg/pom.xml | 38 --- pom.xml | 67 ++-- slider-agent/pom.xml| 62 ++- slider-core/pom.xml | 102 +-- slider-funtest/pom.xml | 48 + .../hbase/slider-hbase-provider/pom.xml | 49 + src/test/clusters/script.md | 2 +- 8 files changed, 208 insertions(+), 198 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml -- diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml index 77f0fc9..051ac84 100644 --- a/app-packages/command-logger/application-pkg/pom.xml +++ b/app-packages/command-logger/application-pkg/pom.xml @@ -54,20 +54,8 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - + org.apache.maven.wagon @@ -75,4 +63,28 @@ + + + + rat + + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml -- diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml index b9bbf06..4550ae1 100644 --- a/app-packages/command-logger/slider-pkg/pom.xml +++ b/app-packages/command-logger/slider-pkg/pom.xml @@ -93,20 +93,6 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - @@ -118,5 +104,29 @@ tar + + + + apache-release + + + + +rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml -- diff --git a/pom.xml b/pom.xml index 167b9ef..df8599f 100644 --- a/pom.xml +++ b/pom.xml @@ -349,36 +349,7 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - -**/*.json -**/*.tar -**/build.properties -**/regionservers -**/slaves -**/httpfs-signature.secret -**/dfs.exclude -**/*.iml -**/rat.txt -DISCLAIMER -app-packages/hbase/target/** -target/* - - - - + @@ -1263,6 +1234,42 @@ + rat + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + +**
git commit: SLIDER-86 failsafe not passing down sysprops
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows be57d7a11 -> 2cdb1ba00 SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0 Branch: refs/heads/feature/SLIDER-86-windows Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4 Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:23:54 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:23:54 2014 +0100 -- slider-funtest/pom.xml | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml -- diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml index 124251f..2b55359 100644 --- a/slider-funtest/pom.xml +++ b/slider-funtest/pom.xml @@ -109,6 +109,12 @@ false ${build.redirect.test.output.to.file} + + + slider.bin.dir + ../slider-assembly/target/slider-${project.version}-all/slider-${project.version} + + true true
[1/2] git commit: SLIDER-86 failsafe not passing down sysprops
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows 2cdb1ba00 -> 8f6f15ff6 SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b Branch: refs/heads/feature/SLIDER-86-windows Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:43:39 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:43:39 2014 +0100 -- .../slider/funtest/basic/SyspropsIT.groovy | 50 1 file changed, 50 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy -- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy new file mode 100644 index 000..728920e --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.basic + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup +import org.apache.slider.test.SliderTestUtils +import org.junit.Test + +/** + * Simple tests to verify that the build has been set up: if these + * fail then the arguments to the test run are incomplete. + * + * This deliberately doesn't depend on CommandTestBase, + * so that individual tests fail with more diagnostics + * than the @BeforeClass failing + */ +@Slf4j +class SyspropsIT extends SliderTestUtils{ + + + @Test + public void testDumpSysprops() throws Throwable { +def sysprops = System.properties +TreeSet sorted = new TreeSet(); +sysprops.keys().each { String it -> sorted.add(it)} +sorted.each { String key -> + log.info("$key=\"${sysprops[key]}\"") +} + + } +}
[2/2] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/feature/SLIDER-86-windows Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[3/3] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17 Branch: refs/heads/feature/SLIDER-86-windows Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4 Parents: d6e73cd Author: Steve Loughran Authored: Wed Sep 10 12:13:22 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:13:22 2014 +0100 -- .../main/java/org/apache/slider/client/SliderClient.java | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8d0978c..606a53c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -201,11 +201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe addService(yarnClient); super.serviceInit(conf); - -//here the superclass is inited; getConfig returns a non-null value -sliderFileSystem = new SliderFileSystem(getConfig()); -YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } /** @@ -298,6 +294,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (coreAction.getHadoopServicesRequired()) { // validate the client SliderUtils.validateSliderClientEnvironment(null); + + //here the superclass is inited; getConfig returns a non-null value + sliderFileSystem = new SliderFileSystem(getConfig()); + YARNRegistryClient = + new YARNRegistryClient(yarnClient, getUsername(), getConfig()); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName();
[2/3] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8 Branch: refs/heads/feature/SLIDER-86-windows Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea Parents: 1efc358 Author: Steve Loughran Authored: Wed Sep 10 12:12:42 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:42 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 107 1 file changed, 74 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index cbc0faf..4b078e7 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -18,6 +18,8 @@ import sys import os import subprocess +import time +from threading import Thread CONF = "conf" @@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" SLIDER_CLASSNAME = "org.apache.slider.Slider" DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s" +ON_POSIX = 'posix' in sys.builtin_module_names + +finished = False +DEBUG = True + """ Launches slider +Nonblocking IO on windows is "tricky" ... see +http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python +to explain the code here + """ @@ -66,6 +77,20 @@ def dirMustExist(dirname): raise Exception("Directory does not exist: %s " % dirname) return dirname + +def debug(text): + if DEBUG: print '[DEBUG] ' + text + + +def error(text): + print '[ERROR] ' + text + sys.stdout.flush() + +def info(text): + print text + sys.stdout.flush() + + def read(pipe, line): """ read a char, append to the listing if there is a char that is not \n @@ -86,55 +111,71 @@ def read(pipe, line): return line, False +def print_output(name, src): + """ + Relay the output stream to stdout line by line + :param name: + :param src: source stream + :return: + """ + + debug ("starting printer for %s" % name ) + line = "" + while not finished: +(line, done) = read(src, line) +if done: + info(name +': ' + line) + line = "" + src.close() + + def runProcess(commandline): """ Run a process :param commandline: command line :return:the return code """ - print "ready to exec : %s" % commandline + debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) - stdout = exe.stdout - stderr = exe.stderr - outline = "" - errline = "" + shell=False, + bufsize=1, + close_fds=ON_POSIX) + + t = Thread(target=print_output, args=("stdout", exe.stdout)) + t.daemon = True + t.start() + t2 = Thread(target=print_output, args=("stderr", exe.stderr,)) + t2.daemon = True + t2.start() + + debug("Waiting for completion") while exe.poll() is None: # process is running; grab output and echo every line -outline, done = read(stdout, outline) -if done: - print outline - outline = "" -errline, done = read(stderr, errline) -if done: - print errline - errline = "" - - # get tail - out, err = exe.communicate() - print outline + out.decode() - print errline + err.decode() +time.sleep(1) + debug("completed with exit code : %d" % exe.returncode) return exe.returncode -def which(program): -def is_exe(fpath): -return os.path.isfile(fpath) and os.access(fpath, os.X_OK) -fpath, fname = os.path.split(program) -if fpath: -if is_exe(program): -return program -else: -for path in os.environ["PATH"].split(os.pathsep): -path = path.strip('"') -exe_file = os.path.join(path, program) -if is_exe(exe_file): -return exe_file +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + + fpath, fname = os.path.split(program) + if fpath: +if is_exe(program): + return program + else: +for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): +return ex
[1/3] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows 8f6f15ff6 -> a6526a17f SLIDER-86: slider funtests call cmd /c slider.py on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583 Branch: refs/heads/feature/SLIDER-86-windows Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc Parents: 8f6f15f Author: Steve Loughran Authored: Wed Sep 10 12:12:10 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:10 2014 +0100 -- .../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++- .../apache/slider/funtest/framework/FuntestProperties.groovy | 1 + .../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 3b52912..b715e38 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.util.ExitUtil +import org.apache.hadoop.util.Shell import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys @@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils { public static final File SLIDER_SCRIPT = new File( SLIDER_TAR_DIRECTORY, BIN_SLIDER).canonicalFile + public static final File SLIDER_SCRIPT_PYTHON = new File( + SLIDER_TAR_DIRECTORY, + BIN_SLIDER_PYTHON).canonicalFile public static final File SLIDER_CONF_DIRECTORY = new File( SLIDER_CONF_DIR).canonicalFile public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY, @@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = SLIDER_SCRIPT +SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT + log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy index 1096dfa..730639b 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy @@ -57,6 +57,7 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { String KEY_TEST_CONF_XML = "slider.test.conf.xml" String KEY_TEST_CONF_DIR = "slider.test.conf.dir" String BIN_SLIDER = "bin/slider" + String BIN_SLIDER_PYTHON = "bin/slider.py" String AGENT_INI = "agent.ini" String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 804e791..3d856c7 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -30,6 +30,7 @@ class SliderShell extends Shell { public static final String BASH = '/bin/bash -s' + public static final String CMD = 'cmd /c' /** * Configuration directory, shared across all inst
[2/2] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3 Branch: refs/heads/feature/SLIDER-86-windows Commit: f7906fa3a6a5fe04113832b747cbe378e5949219 Parents: d51ce5a Author: Steve Loughran Authored: Wed Sep 10 12:25:41 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:41 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index 4b078e7..735e09b 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -135,6 +135,7 @@ def runProcess(commandline): :param commandline: command line :return:the return code """ + global finished debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, @@ -156,6 +157,7 @@ def runProcess(commandline): # process is running; grab output and echo every line time.sleep(1) debug("completed with exit code : %d" % exe.returncode) + finished = True return exe.returncode
[1/2] git commit: SLIDER-418 sider version command fails if the config isn't valid
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows a6526a17f -> f7906fa3a SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1 Branch: refs/heads/feature/SLIDER-86-windows Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f Parents: a6526a1 Author: Steve Loughran Authored: Wed Sep 10 12:25:32 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:32 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 37 ++-- 1 file changed, 27 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 606a53c..ef0e63c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -196,9 +196,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderUtils.forceLogin(); SliderUtils.initProcessSecurity(conf); } -//create the YARN client -yarnClient = new SliderYarnClientImpl(); -addService(yarnClient); + super.serviceInit(conf); @@ -292,13 +290,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AbstractActionArgs coreAction = serviceArgs.getCoreAction(); if (coreAction.getHadoopServicesRequired()) { - // validate the client - SliderUtils.validateSliderClientEnvironment(null); - - //here the superclass is inited; getConfig returns a non-null value - sliderFileSystem = new SliderFileSystem(getConfig()); - YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + initHadoopBinding(); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); @@ -353,6 +345,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return exitCode; } + + /** + * Perform everything needed to init the hadoop binding. + * This assumes that the service is already in inited or started state + * @throws IOException + * @throws SliderException + */ + protected void initHadoopBinding() throws IOException, SliderException { +// validate the client +SliderUtils.validateSliderClientEnvironment(null); +//create the YARN client +yarnClient = new SliderYarnClientImpl(); +yarnClient.init(getConfig()); +if (getServiceState() == STATE.STARTED) { + yarnClient.start(); +} +addService(yarnClient); +// create the filesystem +sliderFileSystem = new SliderFileSystem(getConfig()); + +// and the registry +YARNRegistryClient = +new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } + /** * Delete the zookeeper node associated with the calling user and the cluster **/
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows f7906fa3a -> f2270e34b SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34 Branch: refs/heads/feature/SLIDER-86-windows Commit: f2270e34b610f7e7754bc51226f66d30ca55c581 Parents: f7906fa Author: Steve Loughran Authored: Wed Sep 10 13:01:14 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:01:14 2014 +0100 -- .../funtest/framework/CommandTestBase.groovy| 2 +- .../slider/funtest/framework/SliderShell.groovy | 84 ++-- 2 files changed, 77 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index b715e38..216b68d 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT +SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 3d856c7..e49aee4 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory class SliderShell extends Shell { private static final Logger log = LoggerFactory.getLogger(SliderShell.class); - + private static final Logger LOG = log; public static final String BASH = '/bin/bash -s' - public static final String CMD = 'cmd /c' + public static final String CMD = 'cmd' /** * Configuration directory, shared across all instances. Not marked as volatile, @@ -38,7 +38,7 @@ class SliderShell extends Shell { */ public static File confDir; - public static File script; + public static File scriptFile; public static final List slider_classpath_extra = [] @@ -51,8 +51,8 @@ class SliderShell extends Shell { SliderShell(Collection commands) { super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH) assert confDir != null; -assert script != null; -command = script.absolutePath + " " + commands.join(" ") +assert scriptFile != null; +command = scriptFile.absolutePath + " " + commands.join(" ") } /** @@ -67,7 +67,9 @@ class SliderShell extends Shell { ] if (!slider_classpath_extra.empty) { commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA, - SliderUtils.join(slider_classpath_extra, ":", false)) + SliderUtils.join(slider_classpath_extra, + (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"), + false)) } commandLine << command String script = commandLine.join("\n") @@ -77,8 +79,18 @@ class SliderShell extends Shell { return ret; } + /** + * Add an environment variable + * @param var variable + * @param val value (which will be stringified) + * @return an env variable command + */ String env(String var, Object val) { -return "export " + var + "=${val.toString()};" +if (org.apache.hadoop.util.Shell.WINDOWS) { + return "set " + var + "=${val.toString()};" +} else { + return "export " + var + "=${val.toString()};" +} } /** @@ -152,6 +164,62 @@ class SliderShell extends Shell { } } - + /** + * Execute shell script consisting of as many Strings as we
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows f2270e34b -> 49ae17310 SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731 Branch: refs/heads/feature/SLIDER-86-windows Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b Parents: f2270e3 Author: Steve Loughran Authored: Wed Sep 10 13:05:25 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:05:25 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index e49aee4..6effb46 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -87,7 +87,7 @@ class SliderShell extends Shell { */ String env(String var, Object val) { if (org.apache.hadoop.util.Shell.WINDOWS) { - return "set " + var + "=${val.toString()};" + return "set " + var + "=${val.toString()}" } else { return "export " + var + "=${val.toString()};" }
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows 49ae17310 -> 682ae8a7f SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7 Branch: refs/heads/feature/SLIDER-86-windows Commit: 682ae8a7fa92932f32420486761efa06b5febbc1 Parents: 49ae173 Author: Steve Loughran Authored: Wed Sep 10 13:40:17 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:40:17 2014 +0100 -- .../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 50f3ac0..81d1595 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { log.info "App pkg uploaded at $appPkgPath" } catch (Exception e) { setup_failed = true - fail("Setup failed "+e) + throw e; } }
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows 682ae8a7f -> 16d625e48 SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4 Branch: refs/heads/feature/SLIDER-86-windows Commit: 16d625e4800d2a64b22f62810a832120c09869f3 Parents: 682ae8a Author: Steve Loughran Authored: Wed Sep 10 13:45:35 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:45:35 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index 921adbf..70f119e 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -145,6 +145,7 @@ class FileUploader { public void attemptToCreateHomeDir(String username, Path home) { def privilegedFS = getFileSystemAsUserName(username) +log.info "Creating home dir $home as user ${user.userName} group ${user.primaryGroupName}" privilegedFS.mkdirs(home, new FsPermission((short) 00755)) privilegedFS.setOwner(home, user.userName, user.primaryGroupName) }
[15/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731 Branch: refs/heads/develop Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b Parents: f2270e3 Author: Steve Loughran Authored: Wed Sep 10 13:05:25 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:05:25 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index e49aee4..6effb46 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -87,7 +87,7 @@ class SliderShell extends Shell { */ String env(String var, Object val) { if (org.apache.hadoop.util.Shell.WINDOWS) { - return "set " + var + "=${val.toString()};" + return "set " + var + "=${val.toString()}" } else { return "export " + var + "=${val.toString()};" }
[16/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7 Branch: refs/heads/develop Commit: 682ae8a7fa92932f32420486761efa06b5febbc1 Parents: 49ae173 Author: Steve Loughran Authored: Wed Sep 10 13:40:17 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:40:17 2014 +0100 -- .../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 50f3ac0..81d1595 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { log.info "App pkg uploaded at $appPkgPath" } catch (Exception e) { setup_failed = true - fail("Setup failed "+e) + throw e; } }
[13/19] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3 Branch: refs/heads/develop Commit: f7906fa3a6a5fe04113832b747cbe378e5949219 Parents: d51ce5a Author: Steve Loughran Authored: Wed Sep 10 12:25:41 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:41 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index 4b078e7..735e09b 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -135,6 +135,7 @@ def runProcess(commandline): :param commandline: command line :return:the return code """ + global finished debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, @@ -156,6 +157,7 @@ def runProcess(commandline): # process is running; grab output and echo every line time.sleep(1) debug("completed with exit code : %d" % exe.returncode) + finished = True return exe.returncode
[08/19] git commit: SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode
SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c1d9dc2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c1d9dc2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c1d9dc2 Branch: refs/heads/feature/SLIDER-86-windows Commit: 3c1d9dc2fcec1a897a26388865194c99fc28e2f8 Parents: e24060b Author: Billie Rinaldi Authored: Tue Sep 9 22:14:36 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 9 22:14:36 2014 -0700 -- app-packages/accumulo/README.txt| 17 ++-- app-packages/accumulo/pom.xml | 90 ++-- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++ .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 + .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 + 12 files changed, 294 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/README.txt -- diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt index 634296a..f75814c 100644 --- a/app-packages/accumulo/README.txt +++ b/app-packages/accumulo/README.txt @@ -17,21 +17,28 @@ # How to create a Slider package for Accumulo? - mvn clean package -DskipTests -Paccumulo-app-package + mvn clean package -DskipTests -Paccumulo-app-package-maven + OR + mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \ +-Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball App package can be found in - app-packages/accumulo/target/apache-slider-accumulo-${accumulo.version}-app-package-${slider.version}.zip + app-packages/accumulo/target/slider-accumulo-app-package-*.zip + +In the first case, the version number of the app package will match the +slider version, and in the second case it will match the pkg.version +(intended to be the accumulo version). Verify the content using - zip -Tv apache-slider-accumulo-*.zip + zip -Tv slider-accumulo-app-package*.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So it is advisable that when you create an application package for Slider, include sample/default resources.json and appConfig.json for a minimal Yarn cluster. -The version of Accumulo used for the app package can be adjusted by adding a -flag such as +In the maven packaging case, the version of Accumulo used for the app package +can be adjusted by adding a flag such as -Daccumulo.version=1.5.1 **Note that the LICENSE.txt and NOTICE.txt that are bundled with the app http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/pom.xml -- diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml index 548ced8..f074b01 100644 --- a/app-packages/accumulo/pom.xml +++ b/app-packages/accumulo/pom.xml @@ -30,17 +30,20 @@ package-tmp - ${project.artifactId}-${project.version} +${project.artifactId}-${pkg.version} +${project.build.directory}/${work.dir} +${project.version} +accumulo-${accumulo.version}-bin.tar.gz + +yarn +hadoop + + accumulo-app-package - - - !skipAccumulo - - @@ -64,6 +67,79 @@ org.apache.maven.plugins +maven-antrun-plugin +1.7 + + +copy +validate + + + + + + + run + + + + + + +org.apache.maven.plugins +maven-failsafe-plugin +${maven-failsafe-plugin.version} + + +run-integration-tests + + integration-test + verify + + + + + +true +true +
[07/19] git commit: SLIDER-416 substitute hostname for _HOST in principals
SLIDER-416 substitute hostname for _HOST in principals Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e24060b9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e24060b9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e24060b9 Branch: refs/heads/feature/SLIDER-86-windows Commit: e24060b93751f9a73592be0ca628d3a403d044a3 Parents: ce0c052 Author: Billie Rinaldi Authored: Tue Sep 9 20:17:30 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 9 20:17:30 2014 -0700 -- .../java/org/apache/slider/core/launch/AppMasterLauncher.java | 6 +- .../server/services/security/FsDelegationTokenManager.java | 6 +- 2 files changed, 10 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java -- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index bd8a0a5..c9ac20c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -20,6 +20,7 @@ package org.apache.slider.core.launch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; @@ -35,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.util.Map; import java.util.Set; @@ -196,7 +198,9 @@ public class AppMasterLauncher extends AbstractLauncher { */ private void addSecurityTokens() throws IOException { -String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL); +String tokenRenewer = SecurityUtil.getServerPrincipal( +getConf().get(YarnConfiguration.RM_PRINCIPAL), +InetAddress.getLocalHost().getCanonicalHostName()); if (SliderUtils.isUnset(tokenRenewer)) { throw new IOException( "Can't get Master Kerberos principal for the RM to use as renewer: " http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java index cce2de0..c892b10 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -35,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.security.PrivilegedExceptionAction; import java.text.DateFormat; import java.util.Date; @@ -67,7 +69,9 @@ public class FsDelegationTokenManager { // using HDFS principal... this.remoteUser = UserGroupInformation .loginUserFromKeytabAndReturnUGI( -loginConfig.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), +SecurityUtil.getServerPrincipal( + loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY), +InetAddress.getLocalHost().getCanonicalHostName()), loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); log.info("Created remote user {}. UGI reports current user is {}", this.remoteUser, UserGroupInformation.getCurrentUser());
[19/19] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/develop Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[09/19] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
SLIDER-86: slider funtests call cmd /c slider.py on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583 Branch: refs/heads/develop Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc Parents: 8f6f15f Author: Steve Loughran Authored: Wed Sep 10 12:12:10 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:10 2014 +0100 -- .../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++- .../apache/slider/funtest/framework/FuntestProperties.groovy | 1 + .../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 3b52912..b715e38 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.util.ExitUtil +import org.apache.hadoop.util.Shell import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys @@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils { public static final File SLIDER_SCRIPT = new File( SLIDER_TAR_DIRECTORY, BIN_SLIDER).canonicalFile + public static final File SLIDER_SCRIPT_PYTHON = new File( + SLIDER_TAR_DIRECTORY, + BIN_SLIDER_PYTHON).canonicalFile public static final File SLIDER_CONF_DIRECTORY = new File( SLIDER_CONF_DIR).canonicalFile public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY, @@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = SLIDER_SCRIPT +SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT + log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy index 1096dfa..730639b 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy @@ -57,6 +57,7 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { String KEY_TEST_CONF_XML = "slider.test.conf.xml" String KEY_TEST_CONF_DIR = "slider.test.conf.dir" String BIN_SLIDER = "bin/slider" + String BIN_SLIDER_PYTHON = "bin/slider.py" String AGENT_INI = "agent.ini" String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 804e791..3d856c7 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -30,6 +30,7 @@ class SliderShell extends Shell { public static final String BASH = '/bin/bash -s' + public static final String CMD = 'cmd /c' /** * Configuration directory, shared across all instances. Not marked as volatile, @@ -48,7 +49,7 @@ class SliderShell extends Shell { * @param commands */ SliderShell(C
[04/19] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b Branch: refs/heads/develop Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:43:39 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:43:39 2014 +0100 -- .../slider/funtest/basic/SyspropsIT.groovy | 50 1 file changed, 50 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy -- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy new file mode 100644 index 000..728920e --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.basic + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup +import org.apache.slider.test.SliderTestUtils +import org.junit.Test + +/** + * Simple tests to verify that the build has been set up: if these + * fail then the arguments to the test run are incomplete. + * + * This deliberately doesn't depend on CommandTestBase, + * so that individual tests fail with more diagnostics + * than the @BeforeClass failing + */ +@Slf4j +class SyspropsIT extends SliderTestUtils{ + + + @Test + public void testDumpSysprops() throws Throwable { +def sysprops = System.properties +TreeSet sorted = new TreeSet(); +sysprops.keys().each { String it -> sorted.add(it)} +sorted.each { String key -> + log.info("$key=\"${sysprops[key]}\"") +} + + } +}
[11/19] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17 Branch: refs/heads/develop Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4 Parents: d6e73cd Author: Steve Loughran Authored: Wed Sep 10 12:13:22 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:13:22 2014 +0100 -- .../main/java/org/apache/slider/client/SliderClient.java | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8d0978c..606a53c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -201,11 +201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe addService(yarnClient); super.serviceInit(conf); - -//here the superclass is inited; getConfig returns a non-null value -sliderFileSystem = new SliderFileSystem(getConfig()); -YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } /** @@ -298,6 +294,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (coreAction.getHadoopServicesRequired()) { // validate the client SliderUtils.validateSliderClientEnvironment(null); + + //here the superclass is inited; getConfig returns a non-null value + sliderFileSystem = new SliderFileSystem(getConfig()); + YARNRegistryClient = + new YARNRegistryClient(yarnClient, getUsername(), getConfig()); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName();
[18/19] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/feature/SLIDER-86-windows Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[05/19] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/develop Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[10/19] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8 Branch: refs/heads/develop Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea Parents: 1efc358 Author: Steve Loughran Authored: Wed Sep 10 12:12:42 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:42 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 107 1 file changed, 74 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index cbc0faf..4b078e7 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -18,6 +18,8 @@ import sys import os import subprocess +import time +from threading import Thread CONF = "conf" @@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" SLIDER_CLASSNAME = "org.apache.slider.Slider" DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s" +ON_POSIX = 'posix' in sys.builtin_module_names + +finished = False +DEBUG = True + """ Launches slider +Nonblocking IO on windows is "tricky" ... see +http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python +to explain the code here + """ @@ -66,6 +77,20 @@ def dirMustExist(dirname): raise Exception("Directory does not exist: %s " % dirname) return dirname + +def debug(text): + if DEBUG: print '[DEBUG] ' + text + + +def error(text): + print '[ERROR] ' + text + sys.stdout.flush() + +def info(text): + print text + sys.stdout.flush() + + def read(pipe, line): """ read a char, append to the listing if there is a char that is not \n @@ -86,55 +111,71 @@ def read(pipe, line): return line, False +def print_output(name, src): + """ + Relay the output stream to stdout line by line + :param name: + :param src: source stream + :return: + """ + + debug ("starting printer for %s" % name ) + line = "" + while not finished: +(line, done) = read(src, line) +if done: + info(name +': ' + line) + line = "" + src.close() + + def runProcess(commandline): """ Run a process :param commandline: command line :return:the return code """ - print "ready to exec : %s" % commandline + debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) - stdout = exe.stdout - stderr = exe.stderr - outline = "" - errline = "" + shell=False, + bufsize=1, + close_fds=ON_POSIX) + + t = Thread(target=print_output, args=("stdout", exe.stdout)) + t.daemon = True + t.start() + t2 = Thread(target=print_output, args=("stderr", exe.stderr,)) + t2.daemon = True + t2.start() + + debug("Waiting for completion") while exe.poll() is None: # process is running; grab output and echo every line -outline, done = read(stdout, outline) -if done: - print outline - outline = "" -errline, done = read(stderr, errline) -if done: - print errline - errline = "" - - # get tail - out, err = exe.communicate() - print outline + out.decode() - print errline + err.decode() +time.sleep(1) + debug("completed with exit code : %d" % exe.returncode) return exe.returncode -def which(program): -def is_exe(fpath): -return os.path.isfile(fpath) and os.access(fpath, os.X_OK) -fpath, fname = os.path.split(program) -if fpath: -if is_exe(program): -return program -else: -for path in os.environ["PATH"].split(os.pathsep): -path = path.strip('"') -exe_file = os.path.join(path, program) -if is_exe(exe_file): -return exe_file +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + + fpath, fname = os.path.split(program) + if fpath: +if is_exe(program): + return program + else: +for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): +return exe_file -retu
[14/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34 Branch: refs/heads/develop Commit: f2270e34b610f7e7754bc51226f66d30ca55c581 Parents: f7906fa Author: Steve Loughran Authored: Wed Sep 10 13:01:14 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:01:14 2014 +0100 -- .../funtest/framework/CommandTestBase.groovy| 2 +- .../slider/funtest/framework/SliderShell.groovy | 84 ++-- 2 files changed, 77 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index b715e38..216b68d 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT +SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 3d856c7..e49aee4 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory class SliderShell extends Shell { private static final Logger log = LoggerFactory.getLogger(SliderShell.class); - + private static final Logger LOG = log; public static final String BASH = '/bin/bash -s' - public static final String CMD = 'cmd /c' + public static final String CMD = 'cmd' /** * Configuration directory, shared across all instances. Not marked as volatile, @@ -38,7 +38,7 @@ class SliderShell extends Shell { */ public static File confDir; - public static File script; + public static File scriptFile; public static final List slider_classpath_extra = [] @@ -51,8 +51,8 @@ class SliderShell extends Shell { SliderShell(Collection commands) { super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH) assert confDir != null; -assert script != null; -command = script.absolutePath + " " + commands.join(" ") +assert scriptFile != null; +command = scriptFile.absolutePath + " " + commands.join(" ") } /** @@ -67,7 +67,9 @@ class SliderShell extends Shell { ] if (!slider_classpath_extra.empty) { commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA, - SliderUtils.join(slider_classpath_extra, ":", false)) + SliderUtils.join(slider_classpath_extra, + (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"), + false)) } commandLine << command String script = commandLine.join("\n") @@ -77,8 +79,18 @@ class SliderShell extends Shell { return ret; } + /** + * Add an environment variable + * @param var variable + * @param val value (which will be stringified) + * @return an env variable command + */ String env(String var, Object val) { -return "export " + var + "=${val.toString()};" +if (org.apache.hadoop.util.Shell.WINDOWS) { + return "set " + var + "=${val.toString()};" +} else { + return "export " + var + "=${val.toString()};" +} } /** @@ -152,6 +164,62 @@ class SliderShell extends Shell { } } - + /** + * Execute shell script consisting of as many Strings as we have arguments, + * possibly under an explicit username (requires sudoers privileges). + * NOTE: individual strings are conc
[12/19] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1 Branch: refs/heads/develop Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f Parents: a6526a1 Author: Steve Loughran Authored: Wed Sep 10 12:25:32 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:32 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 37 ++-- 1 file changed, 27 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 606a53c..ef0e63c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -196,9 +196,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderUtils.forceLogin(); SliderUtils.initProcessSecurity(conf); } -//create the YARN client -yarnClient = new SliderYarnClientImpl(); -addService(yarnClient); + super.serviceInit(conf); @@ -292,13 +290,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AbstractActionArgs coreAction = serviceArgs.getCoreAction(); if (coreAction.getHadoopServicesRequired()) { - // validate the client - SliderUtils.validateSliderClientEnvironment(null); - - //here the superclass is inited; getConfig returns a non-null value - sliderFileSystem = new SliderFileSystem(getConfig()); - YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + initHadoopBinding(); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); @@ -353,6 +345,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return exitCode; } + + /** + * Perform everything needed to init the hadoop binding. + * This assumes that the service is already in inited or started state + * @throws IOException + * @throws SliderException + */ + protected void initHadoopBinding() throws IOException, SliderException { +// validate the client +SliderUtils.validateSliderClientEnvironment(null); +//create the YARN client +yarnClient = new SliderYarnClientImpl(); +yarnClient.init(getConfig()); +if (getServiceState() == STATE.STARTED) { + yarnClient.start(); +} +addService(yarnClient); +// create the filesystem +sliderFileSystem = new SliderFileSystem(getConfig()); + +// and the registry +YARNRegistryClient = +new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } + /** * Delete the zookeeper node associated with the calling user and the cluster **/
[06/19] git commit: SLIDER-413 fixed renewal logic to read token expiry time
SLIDER-413 fixed renewal logic to read token expiry time Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce0c0527 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce0c0527 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce0c0527 Branch: refs/heads/feature/SLIDER-86-windows Commit: ce0c05278aa4d7718e3005d756cabff747f3fc5a Parents: 809dc78 Author: Jon Maron Authored: Tue Sep 9 18:14:09 2014 -0400 Committer: Jon Maron Committed: Tue Sep 9 18:14:09 2014 -0400 -- .../security/FsDelegationTokenManager.java | 33 +--- .../TestFsDelegationTokenManager.groovy | 3 +- 2 files changed, 30 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce0c0527/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java index 15d858c..cce2de0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java @@ -22,6 +22,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.util.Time; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.server.appmaster.SliderAppMaster; @@ -34,6 +36,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.text.DateFormat; +import java.util.Date; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -129,6 +133,7 @@ public class FsDelegationTokenManager { class RenewAction extends AsyncAction { Configuration configuration; Token token; +private long tokenExpiryTime; private final FileSystem fs; RenewAction(String name, @@ -145,14 +150,23 @@ public class FsDelegationTokenManager { public Token run() throws Exception { log.info("Obtaining HDFS delgation token with user {}", remoteUser.getShortUserName()); -return fs.getDelegationToken( +Token token = fs.getDelegationToken( remoteUser.getShortUserName()); +tokenExpiryTime = getTokenExpiryTime(token); +log.info("Initial delegation token obtained with expiry time of {}", getPrintableExirationTime(tokenExpiryTime)); +return token; } }); } log.info("Initial request returned delegation token {}", token); } +private long getTokenExpiryTime(Token token) throws IOException { + AbstractDelegationTokenIdentifier id = + (AbstractDelegationTokenIdentifier)token.decodeIdentifier(); + return id.getMaxDate(); +} + protected FileSystem getFileSystem() throws IOException, InterruptedException { // return non-cache FS reference @@ -177,11 +191,12 @@ public class FsDelegationTokenManager { @Override public Long run() throws Exception { long expires = token.renew(fs.getConf()); -log.info("HDFS delegation token renewed"); +log.info("HDFS delegation token renewed. Renewal cycle ends at {}", + getPrintableExirationTime(expires)); return expires; } }); -long calculatedInterval = expires - Time.now(); +long calculatedInterval = tokenExpiryTime - Time.now(); if ( calculatedInterval < renewInterval ) { // time to get a new token since the token will expire before // next renewal interval. Could modify this to be closer to expiry @@ -201,6 +216,11 @@ public class FsDelegationTokenManager { } } +private String getPrintableExirationTime(long expires) { + Date d = new Date(expires); + return DateFormat.getDateTimeInstance().format(d); +} + private void getNewToken() throws InterruptedException, IOException { try { @@ -215,10 +235,13 @@ public class FsDelegationTokenManager { throw new IOException("addDelegationTokens returned no tokens");
[01/19] git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
Repository: incubator-slider Updated Branches: refs/heads/develop 3c1d9dc2f -> 4a0b783ba refs/heads/feature/SLIDER-86-windows 16d625e48 -> 4a0b783ba SLIDER-414 RAT is creating a dir that cannot be deleted Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1 Branch: refs/heads/develop Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef Parents: 65d17d8 Author: Steve Loughran Authored: Tue Sep 9 18:26:52 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 18:26:52 2014 +0100 -- .../command-logger/application-pkg/pom.xml | 38 --- app-packages/command-logger/slider-pkg/pom.xml | 38 --- pom.xml | 67 ++-- slider-agent/pom.xml| 62 ++- slider-core/pom.xml | 102 +-- slider-funtest/pom.xml | 48 + .../hbase/slider-hbase-provider/pom.xml | 49 + src/test/clusters/script.md | 2 +- 8 files changed, 208 insertions(+), 198 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml -- diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml index 77f0fc9..051ac84 100644 --- a/app-packages/command-logger/application-pkg/pom.xml +++ b/app-packages/command-logger/application-pkg/pom.xml @@ -54,20 +54,8 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - + org.apache.maven.wagon @@ -75,4 +63,28 @@ + + + + rat + + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml -- diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml index b9bbf06..4550ae1 100644 --- a/app-packages/command-logger/slider-pkg/pom.xml +++ b/app-packages/command-logger/slider-pkg/pom.xml @@ -93,20 +93,6 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - @@ -118,5 +104,29 @@ tar + + + + apache-release + + + + +rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml -- diff --git a/pom.xml b/pom.xml index 167b9ef..df8599f 100644 --- a/pom.xml +++ b/pom.xml @@ -349,36 +349,7 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - -**/*.json -**/*.tar -**/build.properties -**/regionservers -**/slaves -**/httpfs-signature.secret -**/dfs.exclude -**/*.iml -**/rat.txt -DISCLAIMER -app-packages/hbase/target/** -target/* - - - - + @@ -1263,6 +1234,42 @@ + rat + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + +
[02/19] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0 Branch: refs/heads/develop Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4 Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:23:54 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:23:54 2014 +0100 -- slider-funtest/pom.xml | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml -- diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml index 124251f..2b55359 100644 --- a/slider-funtest/pom.xml +++ b/slider-funtest/pom.xml @@ -109,6 +109,12 @@ false ${build.redirect.test.output.to.file} + + + slider.bin.dir + ../slider-assembly/target/slider-${project.version}-all/slider-${project.version} + + true true
[03/19] git commit: SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted)
SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/809dc786 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/809dc786 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/809dc786 Branch: refs/heads/feature/SLIDER-86-windows Commit: 809dc7860933a383b294b523011162c0872e167b Parents: 65d17d8 Author: tedyu Authored: Tue Sep 9 11:27:36 2014 -0700 Committer: tedyu Committed: Tue Sep 9 11:27:36 2014 -0700 -- app-packages/hbase/README.txt | 43 ++-- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml| 317 +++-- app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- 6 files changed, 227 insertions(+), 177 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/README.txt -- diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt index 1d5c4bb..d0e3067 100644 --- a/app-packages/hbase/README.txt +++ b/app-packages/hbase/README.txt @@ -22,10 +22,26 @@ work well as the default configuration for Slider apps. So it is advisable that when you create an application package for Slider, include sample/default resources.json and appConfig.json for a minimal Yarn cluster. -OPTION-I: Use mvn command -OPTION-II: Manual +OPTION-I: Use a downloaded tarball +OPTION-II: Use the tarball from the local repository -** OPTION - I (use mvn command) ** +** OPTION - I ** +To create the app package you will need the HBase tarball and invoke mvn command +with appropriate parameters. + +Command: +mvn clean package -Phbase-app-package -Dpkg.version= + -Dpkg.name= -Dpkg.src= + +Example: +mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2 + -Dpkg.name=hbase-0.98.5-hadoop2-bin.tar.gz + -Dpkg.src=/Users/user1/Downloads/0.98.5-hadoop2 + +App package can be found in + app-packages/hbase/target/slider-hbase-app-package-${pkg.version}.zip + +** OPTION - II ** You need the HBase version available on local maven repo to create the Slider App Package for HBase. Download the tarball for HBase: @@ -47,29 +63,12 @@ After HBase tarball is published locally in maven repository, you can use the fo App package can be found in app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip -Verify the content using - zip -Tv apache-slider-hbase-*.zip - If an HBase version older than 0.98.3 is desired, it must be installed in the local maven repo. A less descriptive file name can be specified with -Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip. -** OPTION - II (manual) ** -The Slider App Package for HBase can also be created manually. - -Download the tarball for HBase: - e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz - -Copy the hbase tarball to package/files - cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files - -Edit appConfig.json/metainfo.xml - Replace 4 occurrences of "${hbase.version}" with the hbase version values such as "0.98.3-hadoop2" - Replace 1 occurrence of "${app.package.name}" with the desired app package name, e.g. "hbase-v098" - -Create a zip package at the root of the package (/app-packages/hbase/) - zip -r hbase-v098.zip . +** Verifying the content ** Verify the content using - zip -Tv hbase-v098.zip + zip -Tv apache-slider-hbase-*.zip http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/appConfig.json -- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index 07d1b4e..3743602 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -3,13 +3,13 @@ "metadata": { }, "global": { -"application.def": "${app.package.name}.zip", +"application.def": "slider-hbase-app-package-${bld.version}.zip", "create.default.zookeeper.node": "true", "java_home": "/usr/jdk64/jdk1.7.0_45", "system_configs": "core-site", "site.global.app_user": "yarn", -"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", +"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}", "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_port": "8667", http://git-wip-us.apache.org/repos/asf/inc
[17/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4 Branch: refs/heads/develop Commit: 16d625e4800d2a64b22f62810a832120c09869f3 Parents: 682ae8a Author: Steve Loughran Authored: Wed Sep 10 13:45:35 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:45:35 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index 921adbf..70f119e 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -145,6 +145,7 @@ class FileUploader { public void attemptToCreateHomeDir(String username, Path home) { def privilegedFS = getFileSystemAsUserName(username) +log.info "Creating home dir $home as user ${user.userName} group ${user.primaryGroupName}" privilegedFS.mkdirs(home, new FsPermission((short) 00755)) privilegedFS.setOwner(home, user.userName, user.primaryGroupName) }
[36/36] git commit: Merge branch 'feature/SLIDER-86-windows' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'feature/SLIDER-86-windows' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/client/SliderClient.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f6bbb8b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f6bbb8b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f6bbb8b8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f6bbb8b8574c111acda2cc51918b00ed85440d60 Parents: 5784c4a 4a0b783 Author: Steve Loughran Authored: Wed Sep 10 16:43:51 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 16:43:51 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 .../command-logger/application-pkg/pom.xml | 38 ++- app-packages/command-logger/slider-pkg/pom.xml | 38 ++- app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 9 +- app-packages/storm/appConfig.json | 8 +- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/service.py | 47 ++- app-packages/storm/package/scripts/storm.py | 9 +- .../storm/package/scripts/yaml_config.py| 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 +- app-packages/storm/resources.json | 2 +- pom.xml | 78 ++--- slider-agent/pom.xml| 62 ++-- slider-assembly/pom.xml | 23 +- slider-assembly/src/main/scripts/slider | 2 + slider-assembly/src/main/scripts/slider.py | 119 --- slider-core/pom.xml | 102 ++ .../org/apache/slider/client/SliderClient.java | 92 +- .../org/apache/slider/common/SliderKeys.java| 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 + .../conf/TemplateInputPropertiesValidator.java | 2 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../providers/agent/AgentProviderService.java | 15 +- .../security/FsDelegationTokenManager.java | 38 ++- .../actions/TestActionInstallPackage.groovy | 157 + .../TestFsDelegationTokenManager.groovy | 3 +- .../agent/TestAgentProviderService.java | 10 +- slider-funtest/pom.xml | 48 +-- .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 7 +- .../funtest/framework/FileUploader.groovy | 1 + .../funtest/framework/FuntestProperties.groovy | 1 + .../slider/funtest/framework/SliderShell.groovy | 85 - .../slider/funtest/basic/SyspropsIT.groovy | 50 +++ .../hbase/slider-hbase-provider/pom.xml | 49 +-- src/test/clusters/script.md | 2 +- 58 files changed, 1394 insertions(+), 533 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java index
[15/36] git commit: SLIDER-409 AgentProviderService#publishLogFolderPaths() makes inefficient use of keySet iterator
SLIDER-409 AgentProviderService#publishLogFolderPaths() makes inefficient use of keySet iterator Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1d083aee Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1d083aee Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1d083aee Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 1d083aee68cb989e7e61913398a46729caed3a05 Parents: bff5ed6 Author: tedyu Authored: Mon Sep 8 17:14:21 2014 -0700 Committer: tedyu Committed: Mon Sep 8 17:14:21 2014 -0700 -- .../slider/providers/agent/AgentProviderService.java | 10 ++ .../services/security/FsDelegationTokenManager.java | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1d083aee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java -- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 063e61c..9d1df61 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -840,8 +840,9 @@ public class AgentProviderService extends AbstractProviderService implements */ protected void publishLogFolderPaths( Map folders, String containerId, String roleName, String hostFqdn) { -for (String key : folders.keySet()) { - workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, key, containerId), folders.get(key)); +for (Map.Entry entry: folders.entrySet()) { + workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, entry.getKey(), containerId), +entry.getValue()); } publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, @@ -1392,8 +1393,9 @@ public class AgentProviderService extends AbstractProviderService implements for (String configType : configurations.keySet()) { Map configBucket = configurations.get(configType); - for (String configName : configBucket.keySet()) { -String configValue = configBucket.get(configName); + for (Map.Entry entry: configBucket.entrySet()) { +String configName = entry.getKey(); +String configValue = entry.getValue(); for (String lookUpKey : allConfigs.keySet()) { if (configValue != null && configValue.contains(lookUpKey)) { configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey)); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1d083aee/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java index 8f0de3d..15d858c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java @@ -215,6 +215,7 @@ public class FsDelegationTokenManager { throw new IOException("addDelegationTokens returned no tokens"); } token = findMatchingToken(service, tokens); +if (token == null) throw new IOException("Can't get new delegation token "); currentUser.addToken(token.getService(), token); log.info("Expired HDFS delegation token replaced and added as credential to current user");
[13/36] git commit: Merge branch 'develop' into feature/slider_install_package
Merge branch 'develop' into feature/slider_install_package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9af04d6d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9af04d6d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9af04d6d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 9af04d6d480080c6b917f2c55bc5569fab2ffab0 Parents: b4c88a4 bff5ed6 Author: Sumit Mohanty Authored: Mon Sep 8 14:37:01 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 14:37:01 2014 -0700 -- app-packages/storm/README.txt| 19 +++ app-packages/storm/appConfig.json| 4 ++-- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py | 6 +++--- app-packages/storm/package/scripts/service.py| 6 +++--- app-packages/storm/package/scripts/storm.py | 9 ++--- .../storm/package/scripts/yaml_config.py | 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 --- app-packages/storm/resources.json| 2 +- pom.xml | 11 ++- 10 files changed, 42 insertions(+), 38 deletions(-) --
[05/36] git commit: SLIDER-405. Minor changes to the rpm and scripts to facilitate easy deployment
SLIDER-405. Minor changes to the rpm and scripts to facilitate easy deployment Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8da15036 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8da15036 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8da15036 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 8da150363571f7ab1a0b018593bc5907d501748d Parents: 486ef84 Author: Sumit Mohanty Authored: Sat Sep 6 08:32:07 2014 -0700 Committer: Sumit Mohanty Committed: Sat Sep 6 08:32:07 2014 -0700 -- slider-assembly/pom.xml| 23 +-- slider-assembly/src/main/scripts/slider| 2 ++ slider-assembly/src/main/scripts/slider.py | 10 +++--- 3 files changed, 30 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8da15036/slider-assembly/pom.xml -- diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml index dbbd722..c2d0ecb 100644 --- a/slider-assembly/pom.xml +++ b/slider-assembly/pom.xml @@ -33,7 +33,7 @@ ${rpm.basedir}/bin ${rpm.basedir}/lib ${rpm.basedir}/agent -mapred +root hadoop src/conf-hdp ${project.build.directory}/lib @@ -287,10 +287,10 @@ ${rpm.agentdir} - true 0755 ${rpm.username} ${rpm.groupname} + false ${project.build.directory}/agent @@ -308,6 +308,7 @@ 0755 ${rpm.username} ${rpm.groupname} + false ${src.agent.ini.dir} @@ -318,6 +319,24 @@ + + + + ${rpm.agentdir} + 0755 + ${rpm.username} + ${rpm.groupname} + true + + + + ${rpm.agentdir}/conf + 0755 + ${rpm.username} + ${rpm.groupname} + true + +
[06/36] git commit: more security changes
more security changes Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2f2d796 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2f2d796 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2f2d796 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b2f2d796b92455bba0dae9bcc1d8dc8fd6a6e7db Parents: f4871c6 Author: Jon Maron Authored: Sun Sep 7 13:53:16 2014 -0400 Committer: Jon Maron Committed: Sun Sep 7 13:53:16 2014 -0400 -- app-packages/storm/package/scripts/params.py| 7 --- app-packages/storm/package/scripts/service.py | 2 +- app-packages/storm/package/scripts/yaml_config.py | 2 +- app-packages/storm/package/templates/storm_jaas.conf.j2 | 11 --- 4 files changed, 10 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/params.py -- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index a8e36b9..e0bf9e4 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -48,15 +48,16 @@ if ganglia_installed: ganglia_port = config['configurations']['global']['ganglia_server_port'] security_enabled = config['configurations']['global']['security_enabled'] -hostname_lowercase = config['hostname'].lower() if security_enabled: + _hostname_lowercase = config['hostname'].lower() _kerberos_domain = config['configurations']['storm-env']['kerberos_domain'] _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name'] _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name'] - storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', hostname_lowercase) + storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] - storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', hostname_lowercase) + storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase) + storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0] storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/service.py -- diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py index 50ac44c..0ec6413 100644 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@ -50,7 +50,7 @@ def service( if name == "rest_api": cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log") else: - cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") + cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1") if params.security_enabled: if name == "nimbus": http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/yaml_config.py -- diff --git a/app-packages/storm/package/scripts/yaml_config.py b/app-packages/storm/package/scripts/yaml_config.py index 0a1ffa9..5f763cc 100644 --- a/app-packages/storm/package/scripts/yaml_config.py +++ b/app-packages/storm/package/scripts/yaml_config.py @@ -38,7 +38,7 @@ def escape_yaml_propetry(value): # if is map {'a':'b'} if re.match('^\w*\{.+\}\w*$', value): - unquouted = True +unquouted = True try: int(value) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/templates/storm_jaas.conf.j2 -- diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2 index e68cd57..a1ba6ea 100644 --- a/app-packages/storm/package/templates/storm_jaas.conf.j2 +++ b/app-packages/storm/package/templates/storm_jaas.conf.j2 @@ -20,8 +2
[28/36] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4 Parents: d6e73cd Author: Steve Loughran Authored: Wed Sep 10 12:13:22 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:13:22 2014 +0100 -- .../main/java/org/apache/slider/client/SliderClient.java | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8d0978c..606a53c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -201,11 +201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe addService(yarnClient); super.serviceInit(conf); - -//here the superclass is inited; getConfig returns a non-null value -sliderFileSystem = new SliderFileSystem(getConfig()); -YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } /** @@ -298,6 +294,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (coreAction.getHadoopServicesRequired()) { // validate the client SliderUtils.validateSliderClientEnvironment(null); + + //here the superclass is inited; getConfig returns a non-null value + sliderFileSystem = new SliderFileSystem(getConfig()); + YARNRegistryClient = + new YARNRegistryClient(yarnClient, getUsername(), getConfig()); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName();
[30/36] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f7906fa3a6a5fe04113832b747cbe378e5949219 Parents: d51ce5a Author: Steve Loughran Authored: Wed Sep 10 12:25:41 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:41 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index 4b078e7..735e09b 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -135,6 +135,7 @@ def runProcess(commandline): :param commandline: command line :return:the return code """ + global finished debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, @@ -156,6 +157,7 @@ def runProcess(commandline): # process is running; grab output and echo every line time.sleep(1) debug("completed with exit code : %d" % exe.returncode) + finished = True return exe.returncode
[18/36] git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
SLIDER-414 RAT is creating a dir that cannot be deleted Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef Parents: 65d17d8 Author: Steve Loughran Authored: Tue Sep 9 18:26:52 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 18:26:52 2014 +0100 -- .../command-logger/application-pkg/pom.xml | 38 --- app-packages/command-logger/slider-pkg/pom.xml | 38 --- pom.xml | 67 ++-- slider-agent/pom.xml| 62 ++- slider-core/pom.xml | 102 +-- slider-funtest/pom.xml | 48 + .../hbase/slider-hbase-provider/pom.xml | 49 + src/test/clusters/script.md | 2 +- 8 files changed, 208 insertions(+), 198 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml -- diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml index 77f0fc9..051ac84 100644 --- a/app-packages/command-logger/application-pkg/pom.xml +++ b/app-packages/command-logger/application-pkg/pom.xml @@ -54,20 +54,8 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - + org.apache.maven.wagon @@ -75,4 +63,28 @@ + + + + rat + + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml -- diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml index b9bbf06..4550ae1 100644 --- a/app-packages/command-logger/slider-pkg/pom.xml +++ b/app-packages/command-logger/slider-pkg/pom.xml @@ -93,20 +93,6 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - @@ -118,5 +104,29 @@ tar + + + + apache-release + + + + +rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml -- diff --git a/pom.xml b/pom.xml index 167b9ef..df8599f 100644 --- a/pom.xml +++ b/pom.xml @@ -349,36 +349,7 @@ - -org.apache.rat -apache-rat-plugin -${apache-rat-plugin.version} - - -check-licenses - - check - - - - - -**/*.json -**/*.tar -**/build.properties -**/regionservers -**/slaves -**/httpfs-signature.secret -**/dfs.exclude -**/*.iml -**/rat.txt -DISCLAIMER -app-packages/hbase/target/** -target/* - - - - + @@ -1263,6 +1234,42 @@ + rat + + + +org.apache.rat +apache-rat-plugin +${apache-rat-plugin.version} + + +check-licenses + + check + + + + + +**/*.json +**/*.tar +**/build.properties +
[31/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f2270e34b610f7e7754bc51226f66d30ca55c581 Parents: f7906fa Author: Steve Loughran Authored: Wed Sep 10 13:01:14 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:01:14 2014 +0100 -- .../funtest/framework/CommandTestBase.groovy| 2 +- .../slider/funtest/framework/SliderShell.groovy | 84 ++-- 2 files changed, 77 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index b715e38..216b68d 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT +SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 3d856c7..e49aee4 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory class SliderShell extends Shell { private static final Logger log = LoggerFactory.getLogger(SliderShell.class); - + private static final Logger LOG = log; public static final String BASH = '/bin/bash -s' - public static final String CMD = 'cmd /c' + public static final String CMD = 'cmd' /** * Configuration directory, shared across all instances. Not marked as volatile, @@ -38,7 +38,7 @@ class SliderShell extends Shell { */ public static File confDir; - public static File script; + public static File scriptFile; public static final List slider_classpath_extra = [] @@ -51,8 +51,8 @@ class SliderShell extends Shell { SliderShell(Collection commands) { super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH) assert confDir != null; -assert script != null; -command = script.absolutePath + " " + commands.join(" ") +assert scriptFile != null; +command = scriptFile.absolutePath + " " + commands.join(" ") } /** @@ -67,7 +67,9 @@ class SliderShell extends Shell { ] if (!slider_classpath_extra.empty) { commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA, - SliderUtils.join(slider_classpath_extra, ":", false)) + SliderUtils.join(slider_classpath_extra, + (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"), + false)) } commandLine << command String script = commandLine.join("\n") @@ -77,8 +79,18 @@ class SliderShell extends Shell { return ret; } + /** + * Add an environment variable + * @param var variable + * @param val value (which will be stringified) + * @return an env variable command + */ String env(String var, Object val) { -return "export " + var + "=${val.toString()};" +if (org.apache.hadoop.util.Shell.WINDOWS) { + return "set " + var + "=${val.toString()};" +} else { + return "export " + var + "=${val.toString()};" +} } /** @@ -152,6 +164,62 @@ class SliderShell extends Shell { } } - + /** + * Execute shell script consisting of as many Strings as we have arguments, + * possibly under an explicit username (requires sudoers privilege
[02/36] git commit: Temporary Storm package with Security
Temporary Storm package with Security Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0878e528 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0878e528 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0878e528 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 0878e528c78147de0aab4b251ca43afe5943e57b Parents: eb6620e Author: Sumit Mohanty Authored: Thu Aug 28 18:30:42 2014 -0700 Committer: Sumit Mohanty Committed: Thu Aug 28 18:30:42 2014 -0700 -- app-packages/storm/README.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0878e528/app-packages/storm/README.txt -- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index 77474f1..2bae3d2 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -22,8 +22,8 @@ Various configurations provided in this sample are customized for apache-storm-0 So if you use a different version you may need to edit a few config values. Replace the placeholder tarball for Storm. - cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/ - rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE + cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/ + rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE Create a zip package at the root of the package (/app-packages/storm-v0_91/) zip -r Apache_Storm_v_0_9_3.zip .
[10/36] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd16210 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd16210 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd16210 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 3dd16210520ac6cb25fd3c1eb67e3d3b7640ca0b Parents: f66c0cb Author: tedyu Authored: Mon Sep 8 09:18:17 2014 -0700 Committer: tedyu Committed: Mon Sep 8 09:18:17 2014 -0700 -- pom.xml | 7 --- 1 file changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd16210/pom.xml -- diff --git a/pom.xml b/pom.xml index d9b3813..c8607ff 100644 --- a/pom.xml +++ b/pom.xml @@ -1311,13 +1311,6 @@ - hadoop-2.4.1 - -2.4.1 - - - - branch-2 2.6.0-SNAPSHOT
[19/36] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4 Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:23:54 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:23:54 2014 +0100 -- slider-funtest/pom.xml | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml -- diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml index 124251f..2b55359 100644 --- a/slider-funtest/pom.xml +++ b/slider-funtest/pom.xml @@ -109,6 +109,12 @@ false ${build.redirect.test.output.to.file} + + + slider.bin.dir + ../slider-assembly/target/slider-${project.version}-all/slider-${project.version} + + true true
[03/36] git commit: cheanges for secure deployment
cheanges for secure deployment Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f4871c6f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f4871c6f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f4871c6f Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f4871c6fcd2f093a313933de63107bd7e5c372b3 Parents: 0878e52 Author: Jon Maron Authored: Tue Sep 2 19:22:27 2014 -0400 Committer: Jon Maron Committed: Tue Sep 2 19:22:27 2014 -0400 -- app-packages/storm/package/scripts/params.py | 9 - app-packages/storm/package/scripts/service.py | 4 ++-- app-packages/storm/package/scripts/storm.py | 9 ++--- app-packages/storm/package/scripts/yaml_config.py | 10 +- 4 files changed, 21 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/params.py -- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index 93c6018..a8e36b9 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -47,17 +47,16 @@ if ganglia_installed: ganglia_server = config['configurations']['global']['ganglia_server_host'] ganglia_port = config['configurations']['global']['ganglia_server_port'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['global']['security_enabled'] +hostname_lowercase = config['hostname'].lower() if security_enabled: - _hostname_lowercase = config['hostname'].lower() _kerberos_domain = config['configurations']['storm-env']['kerberos_domain'] _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name'] _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name'] - storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) + storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', hostname_lowercase) storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] - storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase) + storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', hostname_lowercase) storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/service.py -- diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py index dc3c733..50ac44c 100644 --- a/app-packages/storm/package/scripts/service.py +++ b/app-packages/storm/package/scripts/service.py @@ -54,10 +54,10 @@ def service( if params.security_enabled: if name == "nimbus": -Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"), +Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_jaas_server_principal}"), user=params.storm_user) else: -Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"), +Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_jaas_client_principal}"), user=params.storm_user) Execute(cmd, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/storm.py -- diff --git a/app-packages/storm/package/scripts/storm.py b/app-packages/storm/package/scripts/storm.py index efca073..8ecb3a1 100644 --- a/app-packages/storm/package/scripts/storm.py +++ b/app-packages/storm/package/scripts/storm.py @@ -50,6 +50,9 @@ def storm(): ) if params.security_enabled: -TemplateConfig( format("{conf_dir}/storm_jaas.conf"), - owner = params.storm_user -) \ No newline at end of file +File(format("{conf_dir}/storm_jaas.conf"), + content=Template("storm_jaas.conf.j2"), + owner = params.storm_user, + group = params.user_group +) + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/yaml_config.py
[07/36] git commit: jaas fix for StormServer
jaas fix for StormServer Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4b2f8aad Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4b2f8aad Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4b2f8aad Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4b2f8aad8d924cd1f2396b0057c885028210310b Parents: b2f2d79 8da1503 Author: Jon Maron Authored: Mon Sep 8 09:27:56 2014 -0400 Committer: Jon Maron Committed: Mon Sep 8 09:27:56 2014 -0400 -- README.md | 4 +- app-packages/accumulo/appConfig.json| 7 +- .../src/test/resources/appConfig_kerberos.json | 57 ++ .../test/resources/appConfig_monitor_ssl.json | 7 +- .../src/test/resources/appConfig_ssl.json | 7 +- app-packages/hbase/appConfig.json | 76 +-- app-packages/hbase/pom.xml | 2 +- app-packages/storm/README.txt | 2 +- app-packages/storm/appConfig.json | 13 +- app-packages/storm/metainfo.xml | 26 +- ...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 - app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/service.py | 43 +- .../storm/package/scripts/status_params.py | 4 +- .../storm/package/templates/config.yaml.j2 | 9 - app-packages/storm/pom.xml | 90 +++ app-packages/storm/resources.json | 12 +- app-packages/storm/src/assembly/storm.xml | 68 +++ pom.xml | 42 +- slider-agent/pom.xml| 4 +- .../src/main/python/agent/Controller.py | 1 + .../python/agent/CustomServiceOrchestrator.py | 10 +- slider-agent/src/main/python/agent/Register.py | 5 +- slider-agent/src/main/python/setup.py | 2 +- .../src/test/python/agent/TestRegistration.py | 5 +- slider-agent/src/test/python/python-wrap| 40 -- slider-agent/src/test/python/python-wrap.sh | 42 ++ slider-agent/src/test/python/unitTests.py | 36 +- slider-assembly/pom.xml | 23 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy| 4 +- slider-assembly/src/main/scripts/slider | 2 + slider-assembly/src/main/scripts/slider.py | 12 +- slider-core/pom.xml | 4 +- .../org/apache/slider/api/InternalKeys.java | 11 + .../org/apache/slider/api/ResourceKeys.java | 13 - .../org/apache/slider/client/SliderClient.java | 43 +- .../common/params/AbstractActionArgs.java | 11 + .../slider/common/params/ActionFreezeArgs.java | 2 +- .../slider/common/params/ActionHelpArgs.java| 14 +- .../slider/common/params/ActionVersionArgs.java | 12 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 8 +- .../apache/slider/common/tools/SliderUtils.java | 571 ++- .../slider/core/build/InstanceBuilder.java | 5 +- .../apache/slider/core/zk/ZookeeperUtils.java | 3 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 189 +++--- .../server/appmaster/RoleLaunchService.java | 8 +- .../server/appmaster/SliderAppMaster.java | 71 ++- .../server/appmaster/actions/ActionHalt.java| 6 +- .../appmaster/actions/ActionKillContainer.java | 20 +- .../appmaster/actions/ActionStartContainer.java | 8 +- .../appmaster/actions/ActionStopSlider.java | 6 +- .../server/appmaster/actions/AsyncAction.java | 47 +- .../actions/ProviderReportedContainerLoss.java | 13 +- .../actions/ProviderStartupCompleted.java | 4 - .../server/appmaster/actions/QueueAccess.java | 5 + .../server/appmaster/actions/QueueService.java | 11 +- .../actions/RegisterComponentInstance.java | 5 + .../actions/ReviewAndFlexApplicationSize.java | 43 ++ .../actions/UnregisterComponentInstance.java| 7 +- .../server/appmaster/monkey/ChaosEntry.java | 2 +- .../slider/server/appmaster/state/AppState.java | 11 +- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +- .../appmaster/web/rest/agent/Register.java | 13 + .../services/utility/EndOfServiceWaiter.java| 84 +++ .../services/workflow/ForkedProcessService.java | 21 +- .../services/workflow/LongLivedProcess.java | 57 +- .../agent/AgentMiniClusterTestBase.groovy | 9 + .../agent/actions/TestActionExists.groovy | 2 +- .../agent/actions/TestActionVersion.groovy | 5 +- .../agent/freezethaw/TestFreezeCommands.groovy | 10 +- .../TestFreezeThawMasterl
[12/36] git commit: SLIDER-181. Storm package updates (fixing some auto-build issues)
SLIDER-181. Storm package updates (fixing some auto-build issues) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bff5ed6d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bff5ed6d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bff5ed6d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: bff5ed6d2994978a08ea709fcc83d35aa8c94566 Parents: 34954fd Author: Sumit Mohanty Authored: Mon Sep 8 09:49:20 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 09:49:20 2014 -0700 -- app-packages/storm/appConfig.json | 2 +- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/resources.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/appConfig.json -- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index 33e38ac..bb607e9 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -32,7 +32,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", -"site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", +"site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.drpc.port": "0", "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/metainfo.xml -- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index 2c314b4..d201f29 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -130,7 +130,7 @@ tarball -files/apache-storm-0.9.1.2.1.1.0-237.tar.gz +files/apache-storm-${pkg.version}.tar.gz http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/resources.json -- diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json index f14b2c8..edfdc1f 100644 --- a/app-packages/storm/resources.json +++ b/app-packages/storm/resources.json @@ -13,7 +13,7 @@ }, "STORM_UI_SERVER": { "yarn.role.priority": "2", - "yarn.component.instances": "0" + "yarn.component.instances": "1" }, "DRPC_SERVER": { "yarn.role.priority": "3",
[23/36] git commit: SLIDER-413 fixed renewal logic to read token expiry time
SLIDER-413 fixed renewal logic to read token expiry time Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce0c0527 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce0c0527 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce0c0527 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: ce0c05278aa4d7718e3005d756cabff747f3fc5a Parents: 809dc78 Author: Jon Maron Authored: Tue Sep 9 18:14:09 2014 -0400 Committer: Jon Maron Committed: Tue Sep 9 18:14:09 2014 -0400 -- .../security/FsDelegationTokenManager.java | 33 +--- .../TestFsDelegationTokenManager.groovy | 3 +- 2 files changed, 30 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce0c0527/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java index 15d858c..cce2de0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java @@ -22,6 +22,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.util.Time; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.server.appmaster.SliderAppMaster; @@ -34,6 +36,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.text.DateFormat; +import java.util.Date; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -129,6 +133,7 @@ public class FsDelegationTokenManager { class RenewAction extends AsyncAction { Configuration configuration; Token token; +private long tokenExpiryTime; private final FileSystem fs; RenewAction(String name, @@ -145,14 +150,23 @@ public class FsDelegationTokenManager { public Token run() throws Exception { log.info("Obtaining HDFS delgation token with user {}", remoteUser.getShortUserName()); -return fs.getDelegationToken( +Token token = fs.getDelegationToken( remoteUser.getShortUserName()); +tokenExpiryTime = getTokenExpiryTime(token); +log.info("Initial delegation token obtained with expiry time of {}", getPrintableExirationTime(tokenExpiryTime)); +return token; } }); } log.info("Initial request returned delegation token {}", token); } +private long getTokenExpiryTime(Token token) throws IOException { + AbstractDelegationTokenIdentifier id = + (AbstractDelegationTokenIdentifier)token.decodeIdentifier(); + return id.getMaxDate(); +} + protected FileSystem getFileSystem() throws IOException, InterruptedException { // return non-cache FS reference @@ -177,11 +191,12 @@ public class FsDelegationTokenManager { @Override public Long run() throws Exception { long expires = token.renew(fs.getConf()); -log.info("HDFS delegation token renewed"); +log.info("HDFS delegation token renewed. Renewal cycle ends at {}", + getPrintableExirationTime(expires)); return expires; } }); -long calculatedInterval = expires - Time.now(); +long calculatedInterval = tokenExpiryTime - Time.now(); if ( calculatedInterval < renewInterval ) { // time to get a new token since the token will expire before // next renewal interval. Could modify this to be closer to expiry @@ -201,6 +216,11 @@ public class FsDelegationTokenManager { } } +private String getPrintableExirationTime(long expires) { + Date d = new Date(expires); + return DateFormat.getDateTimeInstance().format(d); +} + private void getNewToken() throws InterruptedException, IOException { try { @@ -215,10 +235,13 @@ public class FsDelegationTokenManager { throw new IOException("addDelegationTokens ret
[08/36] git commit: SLIDER-407. Add install package command to install app packages
SLIDER-407. Add install package command to install app packages Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b4c88a4b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b4c88a4b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b4c88a4b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b4c88a4b30309fb78f976c783932a133d9fe0058 Parents: 8da1503 Author: Sumit Mohanty Authored: Mon Sep 8 06:36:04 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 06:36:04 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 60 +-- .../org/apache/slider/common/SliderKeys.java| 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 +++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 ++ .../actions/TestActionInstallPackage.groovy | 158 +++ 9 files changed, 296 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef6448d..98074ec 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -48,6 +48,7 @@ import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionEchoArgs; @@ -301,20 +302,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); // actions -if (ACTION_BUILD.equals(action)) { +if (ACTION_INSTALL_PACKAGE.equals(action)) { + exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); +} else if (ACTION_BUILD.equals(action)) { exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs()); } else if (ACTION_CREATE.equals(action)) { exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); } else if (ACTION_FREEZE.equals(action)) { exitCode = actionFreeze(clusterName, - serviceArgs.getActionFreezeArgs()); +serviceArgs.getActionFreezeArgs()); } else if (ACTION_THAW.equals(action)) { exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs()); } else if (ACTION_DESTROY.equals(action)) { exitCode = actionDestroy(clusterName); } else if (ACTION_EXISTS.equals(action)) { exitCode = actionExists(clusterName, - serviceArgs.getActionExistsArgs().live); + serviceArgs.getActionExistsArgs().live); } else if (ACTION_FLEX.equals(action)) { exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs()); } else if (ACTION_GETCONF.equals(action)) { @@ -322,15 +325,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (ACTION_HELP.equals(action) || ACTION_USAGE.equals(action)) { log.info(serviceArgs.usage()); - } else if (ACTION_KILL_CONTAINER.equals(action)) { exitCode = actionKillContainer(clusterName, serviceArgs.getActionKillContainerArgs()); - } else if (ACTION_AM_SUICIDE.equals(action)) { exitCode = actionAmSuicide(clusterName, serviceArgs.getActionAMSuicideArgs()); - } else if (ACTION_LIST.equals(action)) { exitCode = actionList(clusterName); } else if (ACTION_REGISTRY.equals(action)) { @@ -592,6 +592,50 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * Upload application package to user home directory + * + * @param installPkgInfo the arguments needed to build the cluster + * @throws YarnException Yarn problems + * @throws IOException other problems + * @throws BadCommandArgumentsException bad arguments. + */ + public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws + YarnExcep
[17/36] git commit: Merge branch 'feature/slider_install_package' into develop
Merge branch 'feature/slider_install_package' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/65d17d84 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/65d17d84 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/65d17d84 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 65d17d84487594939a904aeefc1b6f85464c35f0 Parents: f6d7917 804788c Author: Sumit Mohanty Authored: Mon Sep 8 22:18:03 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 22:18:03 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 60 +-- .../org/apache/slider/common/SliderKeys.java| 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 +++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 ++ .../actions/TestActionInstallPackage.groovy | 157 +++ 9 files changed, 295 insertions(+), 12 deletions(-) --
[20/36] git commit: SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted)
SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/809dc786 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/809dc786 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/809dc786 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 809dc7860933a383b294b523011162c0872e167b Parents: 65d17d8 Author: tedyu Authored: Tue Sep 9 11:27:36 2014 -0700 Committer: tedyu Committed: Tue Sep 9 11:27:36 2014 -0700 -- app-packages/hbase/README.txt | 43 ++-- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml| 317 +++-- app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- 6 files changed, 227 insertions(+), 177 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/README.txt -- diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt index 1d5c4bb..d0e3067 100644 --- a/app-packages/hbase/README.txt +++ b/app-packages/hbase/README.txt @@ -22,10 +22,26 @@ work well as the default configuration for Slider apps. So it is advisable that when you create an application package for Slider, include sample/default resources.json and appConfig.json for a minimal Yarn cluster. -OPTION-I: Use mvn command -OPTION-II: Manual +OPTION-I: Use a downloaded tarball +OPTION-II: Use the tarball from the local repository -** OPTION - I (use mvn command) ** +** OPTION - I ** +To create the app package you will need the HBase tarball and invoke mvn command +with appropriate parameters. + +Command: +mvn clean package -Phbase-app-package -Dpkg.version= + -Dpkg.name= -Dpkg.src= + +Example: +mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2 + -Dpkg.name=hbase-0.98.5-hadoop2-bin.tar.gz + -Dpkg.src=/Users/user1/Downloads/0.98.5-hadoop2 + +App package can be found in + app-packages/hbase/target/slider-hbase-app-package-${pkg.version}.zip + +** OPTION - II ** You need the HBase version available on local maven repo to create the Slider App Package for HBase. Download the tarball for HBase: @@ -47,29 +63,12 @@ After HBase tarball is published locally in maven repository, you can use the fo App package can be found in app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip -Verify the content using - zip -Tv apache-slider-hbase-*.zip - If an HBase version older than 0.98.3 is desired, it must be installed in the local maven repo. A less descriptive file name can be specified with -Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip. -** OPTION - II (manual) ** -The Slider App Package for HBase can also be created manually. - -Download the tarball for HBase: - e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz - -Copy the hbase tarball to package/files - cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files - -Edit appConfig.json/metainfo.xml - Replace 4 occurrences of "${hbase.version}" with the hbase version values such as "0.98.3-hadoop2" - Replace 1 occurrence of "${app.package.name}" with the desired app package name, e.g. "hbase-v098" - -Create a zip package at the root of the package (/app-packages/hbase/) - zip -r hbase-v098.zip . +** Verifying the content ** Verify the content using - zip -Tv hbase-v098.zip + zip -Tv apache-slider-hbase-*.zip http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/appConfig.json -- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index 07d1b4e..3743602 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -3,13 +3,13 @@ "metadata": { }, "global": { -"application.def": "${app.package.name}.zip", +"application.def": "slider-hbase-app-package-${bld.version}.zip", "create.default.zookeeper.node": "true", "java_home": "/usr/jdk64/jdk1.7.0_45", "system_configs": "core-site", "site.global.app_user": "yarn", -"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}", +"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}", "site.global.ganglia_server_host": "${NN_HOST}", "site.global.ganglia_server_port": "8667", http://git-wip-us
[32/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b Parents: f2270e3 Author: Steve Loughran Authored: Wed Sep 10 13:05:25 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:05:25 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index e49aee4..6effb46 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -87,7 +87,7 @@ class SliderShell extends Shell { */ String env(String var, Object val) { if (org.apache.hadoop.util.Shell.WINDOWS) { - return "set " + var + "=${val.toString()};" + return "set " + var + "=${val.toString()}" } else { return "export " + var + "=${val.toString()};" }
[34/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 16d625e4800d2a64b22f62810a832120c09869f3 Parents: 682ae8a Author: Steve Loughran Authored: Wed Sep 10 13:45:35 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:45:35 2014 +0100 -- .../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index 921adbf..70f119e 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -145,6 +145,7 @@ class FileUploader { public void attemptToCreateHomeDir(String username, Path home) { def privilegedFS = getFileSystemAsUserName(username) +log.info "Creating home dir $home as user ${user.userName} group ${user.primaryGroupName}" privilegedFS.mkdirs(home, new FsPermission((short) 00755)) privilegedFS.setOwner(home, user.userName, user.primaryGroupName) }
[22/36] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[35/36] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[26/36] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
SLIDER-86: slider funtests call cmd /c slider.py on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc Parents: 8f6f15f Author: Steve Loughran Authored: Wed Sep 10 12:12:10 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:10 2014 +0100 -- .../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++- .../apache/slider/funtest/framework/FuntestProperties.groovy | 1 + .../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 3b52912..b715e38 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.util.ExitUtil +import org.apache.hadoop.util.Shell import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys @@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils { public static final File SLIDER_SCRIPT = new File( SLIDER_TAR_DIRECTORY, BIN_SLIDER).canonicalFile + public static final File SLIDER_SCRIPT_PYTHON = new File( + SLIDER_TAR_DIRECTORY, + BIN_SLIDER_PYTHON).canonicalFile public static final File SLIDER_CONF_DIRECTORY = new File( SLIDER_CONF_DIR).canonicalFile public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY, @@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils { log.info "Security is off" } SliderShell.confDir = SLIDER_CONF_DIRECTORY -SliderShell.script = SLIDER_SCRIPT +SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT + log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " + "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy index 1096dfa..730639b 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy @@ -57,6 +57,7 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { String KEY_TEST_CONF_XML = "slider.test.conf.xml" String KEY_TEST_CONF_DIR = "slider.test.conf.dir" String BIN_SLIDER = "bin/slider" + String BIN_SLIDER_PYTHON = "bin/slider.py" String AGENT_INI = "agent.ini" String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 804e791..3d856c7 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -30,6 +30,7 @@ class SliderShell extends Shell { public static final String BASH = '/bin/bash -s' + public static final String CMD = 'cmd /c' /** * Configuration directory, shared across all instances. Not marked as volatile, @@ -48,7 +49,7 @@ class SliderShell extends Shell {
[14/36] git commit: SLIDER-407. Add install package command to install app packages
SLIDER-407. Add install package command to install app packages Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/804788c3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/804788c3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/804788c3 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 804788c338e80381ae33049cdb79f4db25a378ec Parents: 9af04d6 Author: Sumit Mohanty Authored: Mon Sep 8 16:45:08 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 16:45:08 2014 -0700 -- .../src/main/java/org/apache/slider/client/SliderClient.java| 2 +- .../apache/slider/agent/actions/TestActionInstallPackage.groovy | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/804788c3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 98074ec..8d0978c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -628,7 +628,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) { throw new BadCommandArgumentsException("Pkg exists at " + fileInFs.toUri().toString() + - ". Use --replacePkg true to overwrite."); + ". Use --replacePkg to overwrite."); } sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/804788c3/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy index 4d89e9b..88f1298 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy @@ -103,7 +103,7 @@ class TestActionInstallPackage extends AgentMiniClusterTestBase { ) fail("expected an exception, got a status code " + launcher.serviceExitCode) } catch (BadCommandArgumentsException e) { - assert e.message.contains("Use --replacePkg true to overwrite") + assert e.message.contains("Use --replacePkg to overwrite") } } @@ -147,12 +147,11 @@ class TestActionInstallPackage extends AgentMiniClusterTestBase { SliderActions.ACTION_INSTALL_PACKAGE, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", - Arguments.ARG_REPLACE_PKG, true + Arguments.ARG_REPLACE_PKG ], ) } catch (BadCommandArgumentsException e) { log.info(e.message) - assert e.message.contains("Use --replacePkg true to overwrite") } } }
[24/36] git commit: SLIDER-416 substitute hostname for _HOST in principals
SLIDER-416 substitute hostname for _HOST in principals Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e24060b9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e24060b9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e24060b9 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: e24060b93751f9a73592be0ca628d3a403d044a3 Parents: ce0c052 Author: Billie Rinaldi Authored: Tue Sep 9 20:17:30 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 9 20:17:30 2014 -0700 -- .../java/org/apache/slider/core/launch/AppMasterLauncher.java | 6 +- .../server/services/security/FsDelegationTokenManager.java | 6 +- 2 files changed, 10 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java -- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index bd8a0a5..c9ac20c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -20,6 +20,7 @@ package org.apache.slider.core.launch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; @@ -35,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.util.Map; import java.util.Set; @@ -196,7 +198,9 @@ public class AppMasterLauncher extends AbstractLauncher { */ private void addSecurityTokens() throws IOException { -String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL); +String tokenRenewer = SecurityUtil.getServerPrincipal( +getConf().get(YarnConfiguration.RM_PRINCIPAL), +InetAddress.getLocalHost().getCanonicalHostName()); if (SliderUtils.isUnset(tokenRenewer)) { throw new IOException( "Can't get Master Kerberos principal for the RM to use as renewer: " http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java -- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java index cce2de0..c892b10 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -35,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.security.PrivilegedExceptionAction; import java.text.DateFormat; import java.util.Date; @@ -67,7 +69,9 @@ public class FsDelegationTokenManager { // using HDFS principal... this.remoteUser = UserGroupInformation .loginUserFromKeytabAndReturnUGI( -loginConfig.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), +SecurityUtil.getServerPrincipal( + loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY), +InetAddress.getLocalHost().getCanonicalHostName()), loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); log.info("Created remote user {}. UGI reports current user is {}", this.remoteUser, UserGroupInformation.getCurrentUser());
[29/36] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f Parents: a6526a1 Author: Steve Loughran Authored: Wed Sep 10 12:25:32 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:25:32 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 37 ++-- 1 file changed, 27 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 606a53c..ef0e63c 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -196,9 +196,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderUtils.forceLogin(); SliderUtils.initProcessSecurity(conf); } -//create the YARN client -yarnClient = new SliderYarnClientImpl(); -addService(yarnClient); + super.serviceInit(conf); @@ -292,13 +290,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe AbstractActionArgs coreAction = serviceArgs.getCoreAction(); if (coreAction.getHadoopServicesRequired()) { - // validate the client - SliderUtils.validateSliderClientEnvironment(null); - - //here the superclass is inited; getConfig returns a non-null value - sliderFileSystem = new SliderFileSystem(getConfig()); - YARNRegistryClient = - new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + initHadoopBinding(); } int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); @@ -353,6 +345,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return exitCode; } + + /** + * Perform everything needed to init the hadoop binding. + * This assumes that the service is already in inited or started state + * @throws IOException + * @throws SliderException + */ + protected void initHadoopBinding() throws IOException, SliderException { +// validate the client +SliderUtils.validateSliderClientEnvironment(null); +//create the YARN client +yarnClient = new SliderYarnClientImpl(); +yarnClient.init(getConfig()); +if (getServiceState() == STATE.STARTED) { + yarnClient.start(); +} +addService(yarnClient); +// create the filesystem +sliderFileSystem = new SliderFileSystem(getConfig()); + +// and the registry +YARNRegistryClient = +new YARNRegistryClient(yarnClient, getUsername(), getConfig()); + } + /** * Delete the zookeeper node associated with the calling user and the cluster **/
[01/36] git commit: Temporary Storm package with Security
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 5784c4ab5 -> f6bbb8b85 Temporary Storm package with Security Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb6620e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb6620e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb6620e7 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: eb6620e77a1e8c97c5b65f8d098040d03594e478 Parents: 8bfd66e Author: Sumit Mohanty Authored: Thu Aug 28 18:21:18 2014 -0700 Committer: Sumit Mohanty Committed: Thu Aug 28 18:21:18 2014 -0700 -- app-packages/storm/README.txt | 4 +-- app-packages/storm/appConfig.json | 11 app-packages/storm/configuration/storm-env.xml | 25 ++ app-packages/storm/metainfo.xml | 2 +- ...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 ...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 app-packages/storm/package/scripts/params.py| 11 +--- app-packages/storm/package/scripts/service.py | 8 ++ .../storm/package/templates/storm_jaas.conf.j2 | 27 +--- app-packages/storm/resources.json | 2 +- 10 files changed, 91 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/README.txt -- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index 971cf14..77474f1 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -26,10 +26,10 @@ Replace the placeholder tarball for Storm. rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE Create a zip package at the root of the package (/app-packages/storm-v0_91/) - zip -r storm_v091.zip . + zip -r Apache_Storm_v_0_9_3.zip . Verify the content using - unzip -l "$@" storm_v091.zip + unzip -l "$@" Apache_Storm_v_0_9_3.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So its advisable that when you http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/appConfig.json -- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index b8454f9..d6dee84 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -3,12 +3,12 @@ "metadata": { }, "global": { -"application.def": "storm_v091.zip", +"application.def": "Apache_Storm_v_0_9_3.zip", "java_home": "/usr/jdk64/jdk1.7.0_45", "create.default.zookeeper.node": "true", "site.global.app_user": "yarn", -"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237", +"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100", "site.global.user_group": "hadoop", "site.global.security_enabled": "false", "site.global.ganglia_server_host": "${NN_HOST}", @@ -18,13 +18,14 @@ "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}", "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}", +"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", -"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", -"site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", +"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multica
[04/36] git commit: SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python) SLIDER-402. Add support for general purpose cross referencing of na
SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python) SLIDER-402. Add support for general purpose cross referencing of named properties (handle multiple replacements for single property) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/486ef84f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/486ef84f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/486ef84f Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 486ef84f4cb7e6e716e70a82ac08e1c43b3c497d Parents: f14d32c Author: Sumit Mohanty Authored: Fri Sep 5 20:59:04 2014 -0700 Committer: Sumit Mohanty Committed: Fri Sep 5 20:59:24 2014 -0700 -- app-packages/storm/appConfig.json | 6 +-- app-packages/storm/package/scripts/params.py| 2 +- app-packages/storm/package/scripts/service.py | 43 .../providers/agent/AgentProviderService.java | 5 ++- .../agent/TestAgentProviderService.java | 10 ++--- 5 files changed, 28 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/appConfig.json -- diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json index ac9bd49..1eae599 100644 --- a/app-packages/storm/appConfig.json +++ b/app-packages/storm/appConfig.json @@ -23,8 +23,8 @@ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", -"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", -"site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", +"site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", +"site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}", @@ -32,7 +32,7 @@ "site.storm-site.nimbus.host": "${NIMBUS_HOST}", "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]", -"site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", +"site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric
[09/36] git commit: Merge branch 'feature/update_storm_for_93' into develop
Merge branch 'feature/update_storm_for_93' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f66c0cbc Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f66c0cbc Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f66c0cbc Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f66c0cbc0f211da1ea52acf7cad8a20480a1f5ea Parents: 8da1503 4b2f8aa Author: Jon Maron Authored: Mon Sep 8 09:41:14 2014 -0400 Committer: Jon Maron Committed: Mon Sep 8 09:41:14 2014 -0400 -- app-packages/storm/README.txt| 19 +++ app-packages/storm/appConfig.json| 4 ++-- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py | 6 +++--- app-packages/storm/package/scripts/service.py| 6 +++--- app-packages/storm/package/scripts/storm.py | 9 ++--- .../storm/package/scripts/yaml_config.py | 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 --- app-packages/storm/resources.json| 4 ++-- 9 files changed, 41 insertions(+), 30 deletions(-) --
[25/36] git commit: SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode
SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c1d9dc2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c1d9dc2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c1d9dc2 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 3c1d9dc2fcec1a897a26388865194c99fc28e2f8 Parents: e24060b Author: Billie Rinaldi Authored: Tue Sep 9 22:14:36 2014 -0700 Committer: Billie Rinaldi Committed: Tue Sep 9 22:14:36 2014 -0700 -- app-packages/accumulo/README.txt| 17 ++-- app-packages/accumulo/pom.xml | 90 ++-- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++ .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 + .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 + 12 files changed, 294 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/README.txt -- diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt index 634296a..f75814c 100644 --- a/app-packages/accumulo/README.txt +++ b/app-packages/accumulo/README.txt @@ -17,21 +17,28 @@ # How to create a Slider package for Accumulo? - mvn clean package -DskipTests -Paccumulo-app-package + mvn clean package -DskipTests -Paccumulo-app-package-maven + OR + mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \ +-Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball App package can be found in - app-packages/accumulo/target/apache-slider-accumulo-${accumulo.version}-app-package-${slider.version}.zip + app-packages/accumulo/target/slider-accumulo-app-package-*.zip + +In the first case, the version number of the app package will match the +slider version, and in the second case it will match the pkg.version +(intended to be the accumulo version). Verify the content using - zip -Tv apache-slider-accumulo-*.zip + zip -Tv slider-accumulo-app-package*.zip While appConfig.json and resources.json are not required for the package they work well as the default configuration for Slider apps. So it is advisable that when you create an application package for Slider, include sample/default resources.json and appConfig.json for a minimal Yarn cluster. -The version of Accumulo used for the app package can be adjusted by adding a -flag such as +In the maven packaging case, the version of Accumulo used for the app package +can be adjusted by adding a flag such as -Daccumulo.version=1.5.1 **Note that the LICENSE.txt and NOTICE.txt that are bundled with the app http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/pom.xml -- diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml index 548ced8..f074b01 100644 --- a/app-packages/accumulo/pom.xml +++ b/app-packages/accumulo/pom.xml @@ -30,17 +30,20 @@ package-tmp - ${project.artifactId}-${project.version} +${project.artifactId}-${pkg.version} +${project.build.directory}/${work.dir} +${project.version} +accumulo-${accumulo.version}-bin.tar.gz + +yarn +hadoop + + accumulo-app-package - - - !skipAccumulo - - @@ -64,6 +67,79 @@ org.apache.maven.plugins +maven-antrun-plugin +1.7 + + +copy +validate + + + + + + + run + + + + + + +org.apache.maven.plugins +maven-failsafe-plugin +${maven-failsafe-plugin.version} + + +run-integration-tests + + integration-test + verify + + + + + +true +
[33/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 682ae8a7fa92932f32420486761efa06b5febbc1 Parents: 49ae173 Author: Steve Loughran Authored: Wed Sep 10 13:40:17 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:40:17 2014 +0100 -- .../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 50f3ac0..81d1595 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { log.info "App pkg uploaded at $appPkgPath" } catch (Exception e) { setup_failed = true - fail("Setup failed "+e) + throw e; } }
[11/36] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34954fd2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34954fd2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34954fd2 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 34954fd2151d6f0b42877cf9e50aacd7909136bb Parents: 3dd1621 Author: tedyu Authored: Mon Sep 8 09:20:49 2014 -0700 Committer: tedyu Committed: Mon Sep 8 09:20:49 2014 -0700 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34954fd2/pom.xml -- diff --git a/pom.xml b/pom.xml index c8607ff..167b9ef 100644 --- a/pom.xml +++ b/pom.xml @@ -1303,8 +1303,8 @@ local -2.4.1-SNAPSHOT -0.98.4-SNAPSHOT +2.6.0-SNAPSHOT +0.98.5-SNAPSHOT 1.6.0-SNAPSHOT
[27/36] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea Parents: 1efc358 Author: Steve Loughran Authored: Wed Sep 10 12:12:42 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 12:12:42 2014 +0100 -- slider-assembly/src/main/scripts/slider.py | 107 1 file changed, 74 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py -- diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py index cbc0faf..4b078e7 100644 --- a/slider-assembly/src/main/scripts/slider.py +++ b/slider-assembly/src/main/scripts/slider.py @@ -18,6 +18,8 @@ import sys import os import subprocess +import time +from threading import Thread CONF = "conf" @@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" SLIDER_CLASSNAME = "org.apache.slider.Slider" DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s" +ON_POSIX = 'posix' in sys.builtin_module_names + +finished = False +DEBUG = True + """ Launches slider +Nonblocking IO on windows is "tricky" ... see +http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python +to explain the code here + """ @@ -66,6 +77,20 @@ def dirMustExist(dirname): raise Exception("Directory does not exist: %s " % dirname) return dirname + +def debug(text): + if DEBUG: print '[DEBUG] ' + text + + +def error(text): + print '[ERROR] ' + text + sys.stdout.flush() + +def info(text): + print text + sys.stdout.flush() + + def read(pipe, line): """ read a char, append to the listing if there is a char that is not \n @@ -86,55 +111,71 @@ def read(pipe, line): return line, False +def print_output(name, src): + """ + Relay the output stream to stdout line by line + :param name: + :param src: source stream + :return: + """ + + debug ("starting printer for %s" % name ) + line = "" + while not finished: +(line, done) = read(src, line) +if done: + info(name +': ' + line) + line = "" + src.close() + + def runProcess(commandline): """ Run a process :param commandline: command line :return:the return code """ - print "ready to exec : %s" % commandline + debug ("Executing : %s" % commandline) exe = subprocess.Popen(commandline, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) - stdout = exe.stdout - stderr = exe.stderr - outline = "" - errline = "" + shell=False, + bufsize=1, + close_fds=ON_POSIX) + + t = Thread(target=print_output, args=("stdout", exe.stdout)) + t.daemon = True + t.start() + t2 = Thread(target=print_output, args=("stderr", exe.stderr,)) + t2.daemon = True + t2.start() + + debug("Waiting for completion") while exe.poll() is None: # process is running; grab output and echo every line -outline, done = read(stdout, outline) -if done: - print outline - outline = "" -errline, done = read(stderr, errline) -if done: - print errline - errline = "" - - # get tail - out, err = exe.communicate() - print outline + out.decode() - print errline + err.decode() +time.sleep(1) + debug("completed with exit code : %d" % exe.returncode) return exe.returncode -def which(program): -def is_exe(fpath): -return os.path.isfile(fpath) and os.access(fpath, os.X_OK) -fpath, fname = os.path.split(program) -if fpath: -if is_exe(program): -return program -else: -for path in os.environ["PATH"].split(os.pathsep): -path = path.strip('"') -exe_file = os.path.join(path, program) -if is_exe(exe_file): -return exe_file +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + + fpath, fname = os.path.split(program) + if fpath: +if is_exe(program): + return program + else: +for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_
[16/36] git commit: SLIDER-411 Correct error message when property whose name starts with 'yarn.' is encountered in template
SLIDER-411 Correct error message when property whose name starts with 'yarn.' is encountered in template Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f6d7917d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f6d7917d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f6d7917d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f6d7917d250509bd751164a891ef43f770243327 Parents: 1d083ae Author: tedyu Authored: Mon Sep 8 20:50:10 2014 -0700 Committer: tedyu Committed: Mon Sep 8 20:50:10 2014 -0700 -- .../apache/slider/core/conf/TemplateInputPropertiesValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6d7917d/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java -- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java index 5494174..aad2757 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java @@ -27,7 +27,7 @@ public class TemplateInputPropertiesValidator void validatePropertyNamePrefix(String key) throws BadConfigException { if (key.startsWith("yarn.")) { throw new BadConfigException( - "argument %s does not have 'yarn.' prefix", key); + "argument %s has 'yarn.' prefix - this is not allowed in templates", key); } }
[21/36] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e Parents: be57d7a Author: Steve Loughran Authored: Tue Sep 9 19:43:39 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:43:39 2014 +0100 -- .../slider/funtest/basic/SyspropsIT.groovy | 50 1 file changed, 50 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy -- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy new file mode 100644 index 000..728920e --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.basic + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup +import org.apache.slider.test.SliderTestUtils +import org.junit.Test + +/** + * Simple tests to verify that the build has been set up: if these + * fail then the arguments to the test run are incomplete. + * + * This deliberately doesn't depend on CommandTestBase, + * so that individual tests fail with more diagnostics + * than the @BeforeClass failing + */ +@Slf4j +class SyspropsIT extends SliderTestUtils{ + + + @Test + public void testDumpSysprops() throws Throwable { +def sysprops = System.properties +TreeSet sorted = new TreeSet(); +sysprops.keys().each { String it -> sorted.add(it)} +sorted.each { String key -> + log.info("$key=\"${sysprops[key]}\"") +} + + } +}
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows [deleted] 4a0b783ba
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows [created] 8b32aa17a
git commit: SLIDER-381 Slider-client exists command does not report status correctly
Repository: incubator-slider Updated Branches: refs/heads/develop 8b32aa17a -> 7c95d5b05 SLIDER-381 Slider-client exists command does not report status correctly Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c95d5b0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c95d5b0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c95d5b0 Branch: refs/heads/develop Commit: 7c95d5b059045cc01fa9756308f50e01644abb2d Parents: 8b32aa1 Author: Steve Loughran Authored: Wed Sep 10 18:04:18 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 18:04:18 2014 +0100 -- .../java/org/apache/slider/client/SliderClient.java| 13 ++--- 1 file changed, 6 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c95d5b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef0e63c..7c1fd1e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1623,10 +1623,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @return exit code */ @VisibleForTesting - public int actionExists(String name, boolean live) throws YarnException, IOException { + public int actionExists(String name, boolean checkLive) throws YarnException, IOException { verifyBindingsDefined(); SliderUtils.validateClusterName(name); -log.debug("actionExists({}, {})", name, live); +log.debug("actionExists({}, {})", name, checkLive); //initial probe for a cluster in the filesystem Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name); @@ -1636,10 +1636,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe //test for liveness if desired -if (live) { +if (checkLive) { ApplicationReport instance = findInstance(name); if (instance == null) { -log.info("cluster {} not running", name); +log.info("Cluster {} not running", name); return EXIT_FALSE; } else { // the app exists, but it may be in a terminated state @@ -1653,11 +1653,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.debug("State {}", report); return EXIT_FALSE; } -log.info("Cluster {} is running:\n{}", name, report); +log.info("Cluster {} is live:\n{}", name, report); } } else { - log.info("Cluster {} exists but is not running", name); - + log.info("Cluster {} exists", name); } return EXIT_SUCCESS; }
[06/17] git commit: Merge commit '7c95d5b059045cc01fa9756308f50e01644abb2d' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge commit '7c95d5b059045cc01fa9756308f50e01644abb2d' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4eeed33a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4eeed33a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4eeed33a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4eeed33abd8623c094ea771e1d6cd2875c15c1fd Parents: f6bbb8b 7c95d5b Author: Steve Loughran Authored: Thu Sep 11 10:37:11 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 11 10:37:11 2014 +0100 -- .../java/org/apache/slider/client/SliderClient.java| 13 ++--- .../org/apache/slider/common/tools/SliderUtils.java| 4 +++- .../abstracttests/AbstractTestBuildSetup.groovy| 3 +++ 3 files changed, 12 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eeed33a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java --
[05/17] git commit: Merge branch 'develop' into feature/slider_install_package
Merge branch 'develop' into feature/slider_install_package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/57bce09b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/57bce09b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/57bce09b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 57bce09b0f5c45a7e331418216b488a961398d00 Parents: fc3574e fa7130d Author: Sumit Mohanty Authored: Wed Sep 10 19:54:56 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 10 19:54:56 2014 -0700 -- app-packages/accumulo/README.txt| 17 ++- app-packages/accumulo/pom.xml | 90 +-- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++ .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 ++ .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 ++ .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 ++ .../command-logger/application-pkg/pom.xml | 38 --- app-packages/command-logger/slider-pkg/pom.xml | 38 --- app-packages/hbase/metainfo.xml | 2 +- pom.xml | 67 +++- slider-agent/pom.xml| 62 ++- slider-assembly/src/main/scripts/slider.py | 109 +-- slider-core/pom.xml | 102 + .../org/apache/slider/client/SliderClient.java | 51 ++--- .../apache/slider/common/tools/SliderUtils.java | 4 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- slider-funtest/pom.xml | 48 .../abstracttests/AbstractTestBuildSetup.groovy | 3 + .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 7 +- .../funtest/framework/FileUploader.groovy | 1 + .../funtest/framework/FuntestProperties.groovy | 1 + .../slider/funtest/framework/SliderShell.groovy | 85 +-- .../slider/funtest/basic/SyspropsIT.groovy | 50 + .../hbase/slider-hbase-provider/pom.xml | 49 + src/test/clusters/script.md | 2 +- 34 files changed, 795 insertions(+), 291 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java --
[08/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance directory Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3f742d1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3f742d1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3f742d1 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b3f742d1b32dd8043ac9a8e1ef546e938dc832c7 Parents: 57bce09 Author: Sumit Mohanty Authored: Thu Sep 11 07:36:43 2014 -0700 Committer: Sumit Mohanty Committed: Thu Sep 11 07:36:43 2014 -0700 -- .../agent/TestAgentClientProvider.java | 53 .../agent/TestAgentClientProvider2.java | 88 2 files changed, 88 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3f742d1/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java -- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java index 1ad59e3..4cb35aa 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java @@ -18,38 +18,22 @@ package org.apache.slider.providers.agent; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.slider.api.InternalKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.ProviderUtils; import org.apache.slider.tools.TestUtility; -import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.easymock.PowerMock; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Set; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.anyObject; - /** * */ -@RunWith(PowerMockRunner.class) -@PrepareForTest(ProviderUtils.class) public class TestAgentClientProvider { protected static final Logger log = LoggerFactory.getLogger(TestAgentClientProvider.class); @@ -90,41 +74,4 @@ public class TestAgentClientProvider { Assert.assertTrue(e.getMessage().contains("Application definition must be provided")); } } - - @Test - public void testPrepareAMAndConfigForLaunch() throws Exception { -AgentClientProvider provider = new AgentClientProvider(null); -SliderFileSystem sfs = PowerMock.createMock(SliderFileSystem.class); -FileSystem fs = PowerMock.createMock(FileSystem.class); -Configuration serviceConf = PowerMock.createMock(Configuration.class); -PowerMock.mockStatic(ProviderUtils.class); - -expect(sfs.getFileSystem()).andReturn(fs); -expect(fs.mkdirs(anyObject(Path.class))).andReturn(true); -expect(ProviderUtils.addAgentTar( -anyObject(), anyObject(String.class), anyObject(SliderFileSystem.class), anyObject(Path.class))). -andReturn(true); - -AggregateConf instanceDefinition = new AggregateConf(); -ConfTree tree = new ConfTree(); -tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); -instanceDefinition.setInternal(tree); - -PowerMock.replay(sfs, fs, serviceConf, ProviderUtils.class); - -provider.prepareAMAndConfigForLaunch( -sfs, serviceConf, null, instanceDefinition, null, -null, null, null, null, false); - - Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH)); -tree.global.remove(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - -// Verify that slider-agent.tar.gz got added -Path tempPath = new Path(".", "temp"); -provider.prepareAMAndConfigForLaunch( -sfs, serviceConf, null, instanceDefinition, null, -null, null, null, tempPath, false); -PowerMock.verify(sfs, fs, ProviderUtils.class); - Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH)); - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3f742d1/slider-core/src/test/java/org/apache/slider/provi
[04/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance directory Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fc3574e0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fc3574e0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fc3574e0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: fc3574e0fa4db8fdeb94f53c6cbcbfe85d373ae8 Parents: 809dc78 Author: Sumit Mohanty Authored: Wed Sep 10 19:54:19 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 10 19:54:19 2014 -0700 -- slider-assembly/pom.xml | 20 slider-assembly/src/assembly/slider-bin.xml | 8 +++ slider-core/pom.xml | 28 +++ .../org/apache/slider/api/InternalKeys.java | 6 ++- .../org/apache/slider/client/SliderClient.java | 12 ++--- .../org/apache/slider/common/SliderKeys.java| 1 + .../apache/slider/common/tools/SliderUtils.java | 4 ++ .../slider/core/build/InstanceBuilder.java | 28 +-- .../apache/slider/providers/ProviderUtils.java | 33 ++-- .../providers/agent/AgentClientProvider.java| 41 +++ .../providers/agent/AgentProviderService.java | 19 ++- .../providers/agent/TestBuildBasicAgent.groovy | 19 --- .../agent/TestAgentClientProvider.java | 53 13 files changed, 203 insertions(+), 69 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-assembly/pom.xml -- diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml index c2d0ecb..ac395fb 100644 --- a/slider-assembly/pom.xml +++ b/slider-assembly/pom.xml @@ -104,6 +104,26 @@ + + +copy-to-lib +package + + copy + + + + + org.apache.slider + slider-agent + tar.gz + false + ${project.build.directory}/lib + slider-agent.tar.gz + + + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-assembly/src/assembly/slider-bin.xml -- diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml index 6f3b021..c5611f8 100644 --- a/slider-assembly/src/assembly/slider-bin.xml +++ b/slider-assembly/src/assembly/slider-bin.xml @@ -59,6 +59,14 @@ ${project.build.directory}/agent + lib + +slider-agent.tar.gz + + + + + ${project.build.directory}/agent agent slider-agent.tar.gz http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/pom.xml -- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 326af88..96b0f83 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -484,6 +484,34 @@ jetty-sslengine + + org.powermock + powermock-core + 1.5 + test + + + + org.powermock + powermock-reflect + 1.5 + test + + + + org.powermock + powermock-api-easymock + 1.5 + test + + + + org.powermock + powermock-module-junit4 + 1.5 + test + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java -- diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index b58f1c6..ffa3600 100644 --- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -47,7 +47,11 @@ public interface InternalKeys { /** * internal temp directory: {@value} */ - String INTERNAL_AM_TMP_DIR = "internal.tmp.dir"; + String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir"; + /** + * internal temp directory: {@value} + */ + String INTERNAL_TMP_DIR = "internal.tmp.dir"; /** * where a snapshot of the original conf dir is: {@value} */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff -
[01/17] git commit: SLIDER-415 fix funtests against secure cluster
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry f6bbb8b85 -> f67b5507b SLIDER-415 fix funtests against secure cluster Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8b32aa17 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8b32aa17 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8b32aa17 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 8b32aa17a4d9fb237d0418f47c77bfd5dc529dbf Parents: 4a0b783 Author: Jon Maron Authored: Wed Sep 10 10:01:10 2014 -0400 Committer: Jon Maron Committed: Wed Sep 10 10:01:10 2014 -0400 -- .../main/java/org/apache/slider/common/tools/SliderUtils.java| 4 +++- .../slider/funtest/abstracttests/AbstractTestBuildSetup.groovy | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b32aa17/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java -- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 4cdc580..4467849 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1019,7 +1019,9 @@ public final class SliderUtils { if (authUser.getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.SIMPLE) { throw new BadConfigException("Auth User is not Kerberized %s" + - " -security has already been set up with the wrong authentication method", + " -security has already been set up with the wrong authentication method. " + + "This can occur if a file system has already been created prior to the loading of " + + "the security configuration.", authUser); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b32aa17/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy -- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy index e0b87f7..104de58 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy @@ -160,6 +160,9 @@ abstract class AbstractTestBuildSetup extends SliderTestUtils implements Funtest @Test public void testSecuritySettingsValid() throws Throwable { Configuration conf = loadSliderConf(); +if (SliderUtils.isHadoopClusterSecure(conf)) { + UserGroupInformation.setLoginUser(null) +} if (SliderUtils.maybeInitSecurity(conf)) { log.info("Security enabled") SliderUtils.forceLogin()
[14/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance directory Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/219e7f8e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/219e7f8e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/219e7f8e Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 219e7f8e3f2888209934913eaa26972ec446bc4f Parents: db67363 Author: Sumit Mohanty Authored: Thu Sep 11 12:45:16 2014 -0700 Committer: Sumit Mohanty Committed: Thu Sep 11 12:45:16 2014 -0700 -- .../providers/agent/TestAgentProviderService.java | 18 +++--- 1 file changed, 11 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/219e7f8e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java -- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index aa9dd60..b652bb5 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -272,9 +272,10 @@ public class TestAgentProviderService { MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); Path containerTmpDirPath = new Path(".", "test"); -FileSystem mockFs = new MockFileSystem(); +FileSystem mockFs = createNiceMock(FileSystem.class); +expect(mockFs.exists(anyObject(Path.class))).andReturn(true); expect(sliderFileSystem.getFileSystem()) -.andReturn(new FilterFileSystem(mockFs)).anyTimes(); +.andReturn(mockFs).anyTimes(); expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class))) .andReturn(createNiceMock(LocalResource.class)).anyTimes(); @@ -331,7 +332,7 @@ public class TestAgentProviderService { treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); -replay(access, ctx, container, sliderFileSystem); +replay(access, ctx, container, sliderFileSystem, mockFs); try { mockAps.buildContainerLaunchContext(launcher, @@ -423,7 +424,9 @@ public class TestAgentProviderService { AgentProviderService mockAps = Mockito.spy(aps); doReturn(access).when(mockAps).getAmState(); -doReturn("scripts/hbase_master.py").when(mockAps) +CommandScript cs = new CommandScript(); +cs.setScript("scripts/hbase_master.py"); +doReturn(cs).when(mockAps) .getScriptPathFromMetainfo(anyString()); Metainfo metainfo = new Metainfo(); Application application = new Application(); @@ -922,9 +925,10 @@ public class TestAgentProviderService { MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); Path containerTmpDirPath = new Path(".", "test"); -FileSystem mockFs = new MockFileSystem(); +FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class); expect(sliderFileSystem.getFileSystem()) -.andReturn(new FilterFileSystem(mockFs)).anyTimes(); +.andReturn(mockFs).anyTimes(); +expect(mockFs.exists(anyObject(Path.class))).andReturn(true); expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class))) .andReturn(createNiceMock(LocalResource.class)).anyTimes(); @@ -982,7 +986,7 @@ public class TestAgentProviderService { treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes(); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); -replay(access, ctx, container, sliderFileSystem); +replay(access, ctx, container, sliderFileSystem, mockFs); // build two containers try {
[17/17] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f67b5507 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f67b5507 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f67b5507 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f67b5507b7230a025e7c54064e2614258f009116 Parents: 6e9380c a8dfc67 Author: Steve Loughran Authored: Fri Sep 12 09:59:58 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 12 09:59:58 2014 +0100 -- app-packages/accumulo/pom.xml | 52 +++-- .../command-logger/slider-pkg/appConfig.json| 2 +- .../command-logger/slider-pkg/metainfo.xml | 2 +- app-packages/hbase/metainfo.xml | 2 +- slider-assembly/pom.xml | 20 ++ slider-assembly/src/assembly/slider-bin.xml | 8 + slider-core/pom.xml | 28 +++ .../org/apache/slider/api/InternalKeys.java | 6 +- .../org/apache/slider/client/SliderClient.java | 12 +- .../org/apache/slider/common/SliderKeys.java| 1 + .../apache/slider/common/tools/SliderUtils.java | 4 + .../slider/core/build/InstanceBuilder.java | 28 +-- .../apache/slider/providers/ProviderUtils.java | 33 ++- .../providers/agent/AgentClientProvider.java| 41 ++-- .../providers/agent/AgentProviderService.java | 19 +- .../web/rest/agent/ExecutionCommand.java| 16 ++ .../appmaster/web/rest/agent/StatusCommand.java | 13 ++ .../test_command_log/appConfig.json | 2 +- .../providers/agent/TestBuildBasicAgent.groovy | 19 -- .../agent/TestAgentClientProvider2.java | 88 +++ .../agent/TestAgentProviderService.java | 230 ++- slider-funtest/pom.xml | 7 +- .../org/apache/chaos/remote/BuildPidPath.groovy | 54 + .../org/apache/chaos/remote/Clustat.groovy | 64 ++ .../apache/chaos/remote/JschToCommonsLog.groovy | 78 +++ .../chaos/remote/RemoteDaemonOperations.groovy | 60 + .../org/apache/chaos/remote/RemoteServer.groovy | 188 +++ .../org/apache/chaos/remote/SshCommands.groovy | 190 +++ .../funtest/lifecycle/AMFailuresIT.groovy | 191 +++ 29 files changed, 1353 insertions(+), 105 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java index e10eb50,b2d10f6..baa413e --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@@ -1186,10 -1157,12 +1181,12 @@@ public class SliderClient extends Abstr clusterDirectory, generatedConfDirPath, clusterSecure -); + ); - // now add the image if it was set + // TODO: consider supporting apps that don't have an image path + Path imagePath = + SliderUtils.extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java --
[10/17] git commit: SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f1c1f705 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f1c1f705 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f1c1f705 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f1c1f705fe24af7e768fcf706370d0f0bb625f11 Parents: a509561 Author: Sumit Mohanty Authored: Thu Sep 11 10:18:23 2014 -0700 Committer: Sumit Mohanty Committed: Thu Sep 11 10:40:00 2014 -0700 -- .../command-logger/slider-pkg/appConfig.json| 2 +- .../command-logger/slider-pkg/metainfo.xml | 2 +- .../test_command_log/appConfig.json | 2 +- slider-funtest/pom.xml | 7 +- slider-funtest/pom.xml.orig | 357 +++ .../org/apache/chaos/remote/BuildPidPath.groovy | 54 +++ .../org/apache/chaos/remote/Clustat.groovy | 64 .../apache/chaos/remote/JschToCommonsLog.groovy | 78 .../chaos/remote/RemoteDaemonOperations.groovy | 60 .../org/apache/chaos/remote/RemoteServer.groovy | 188 ++ .../org/apache/chaos/remote/SshCommands.groovy | 190 ++ .../funtest/lifecycle/AMFailuresIT.groovy | 191 ++ 12 files changed, 1191 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/appConfig.json -- diff --git a/app-packages/command-logger/slider-pkg/appConfig.json b/app-packages/command-logger/slider-pkg/appConfig.json index bfdffb8..791043f 100644 --- a/app-packages/command-logger/slider-pkg/appConfig.json +++ b/app-packages/command-logger/slider-pkg/appConfig.json @@ -11,7 +11,7 @@ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger", "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install", -"site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log", +"site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log", "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p" }, "components": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/metainfo.xml -- diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml b/app-packages/command-logger/slider-pkg/metainfo.xml index 976fa81..b8de9a3 100644 --- a/app-packages/command-logger/slider-pkg/metainfo.xml +++ b/app-packages/command-logger/slider-pkg/metainfo.xml @@ -42,7 +42,7 @@ tarball -files/command_log.tar +files/command-logger.tar http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-core/src/test/app_packages/test_command_log/appConfig.json -- diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json b/slider-core/src/test/app_packages/test_command_log/appConfig.json index 684924d..7586a0c 100644 --- a/slider-core/src/test/app_packages/test_command_log/appConfig.json +++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json @@ -12,7 +12,7 @@ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger", "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install", -"site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log", +"site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log", "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p" }, "components": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/pom.xml -- diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml index 124251f..55b6351 100644 --- a/slider-funtest/pom.xml +++ b/slider-funtest/pom.xml @@ -121,6 +121,7 @@ apache-slider-command-logger.zip ../slider-core/src/test/app_packages/test_command_log/resources.json ../slider-core/src/test/app_packages/test_command_log/appConfig.json + /Users/gsaha/work/workspace/ambari-vagrant/centos6.4 @@ -288,7 +289,11 @@ org.codehaus
[03/17] git commit: Remove -bin from hbase tar ball name
Remove -bin from hbase tar ball name Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa7130d3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa7130d3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa7130d3 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: fa7130d306dfba1e60a70f57bb1735eb77424872 Parents: 7c95d5b Author: tedyu Authored: Wed Sep 10 12:52:18 2014 -0700 Committer: tedyu Committed: Wed Sep 10 12:52:18 2014 -0700 -- app-packages/hbase/metainfo.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa7130d3/app-packages/hbase/metainfo.xml -- diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml index 10bee97..1072a52 100644 --- a/app-packages/hbase/metainfo.xml +++ b/app-packages/hbase/metainfo.xml @@ -160,7 +160,7 @@ tarball -files/hbase-${pkg.version}-bin.tar.gz +files/hbase-${pkg.version}.tar.gz
[02/17] git commit: SLIDER-381 Slider-client exists command does not report status correctly
SLIDER-381 Slider-client exists command does not report status correctly Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c95d5b0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c95d5b0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c95d5b0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 7c95d5b059045cc01fa9756308f50e01644abb2d Parents: 8b32aa1 Author: Steve Loughran Authored: Wed Sep 10 18:04:18 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 18:04:18 2014 +0100 -- .../java/org/apache/slider/client/SliderClient.java| 13 ++--- 1 file changed, 6 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c95d5b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef0e63c..7c1fd1e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1623,10 +1623,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @return exit code */ @VisibleForTesting - public int actionExists(String name, boolean live) throws YarnException, IOException { + public int actionExists(String name, boolean checkLive) throws YarnException, IOException { verifyBindingsDefined(); SliderUtils.validateClusterName(name); -log.debug("actionExists({}, {})", name, live); +log.debug("actionExists({}, {})", name, checkLive); //initial probe for a cluster in the filesystem Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name); @@ -1636,10 +1636,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe //test for liveness if desired -if (live) { +if (checkLive) { ApplicationReport instance = findInstance(name); if (instance == null) { -log.info("cluster {} not running", name); +log.info("Cluster {} not running", name); return EXIT_FALSE; } else { // the app exists, but it may be in a terminated state @@ -1653,11 +1653,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.debug("State {}", report); return EXIT_FALSE; } -log.info("Cluster {} is running:\n{}", name, report); +log.info("Cluster {} is live:\n{}", name, report); } } else { - log.info("Cluster {} exists but is not running", name); - + log.info("Cluster {} exists", name); } return EXIT_SUCCESS; }