[01/47] git commit: SLIDER-386 fixed role-specific failure threshold
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 6e883bf91 ->
5784c4ab5
SLIDER-386 fixed role-specific failure threshold
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ecab238
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ecab238
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ecab238
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7ecab238a919ae4013f23b259ab4eba317635485
Parents: 82cf1f0
Author: Billie Rinaldi
Authored: Tue Sep 2 08:48:10 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 2 08:48:10 2014 -0700
--
.../java/org/apache/slider/server/appmaster/state/AppState.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ecab238/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 07976ef..8b70185 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1559,7 +1559,7 @@ public class AppState {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
-ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
[15/47] git commit: SLIDER-350 use gawk as awk implementation for windows; add checks for that and xargs on the command line
SLIDER-350 use gawk as awk implementation for windows; add checks for that and
xargs on the command line
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/690793a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/690793a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/690793a4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 690793a4f8a82a3b7395271803ae9af89eb2cdda
Parents: 2e3c4c8
Author: Steve Loughran
Authored: Thu Sep 4 14:00:01 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 4 14:00:01 2014 +0100
--
.../common/tools/TestWindowsSupport.groovy | 38 ++--
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
2 files changed, 37 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 84356ea..d18fe76 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -105,12 +105,33 @@ class TestWindowsSupport extends SliderTestBase {
}
+
+ @Test
+ public void testHasGawkInstalled() throws Throwable {
+assume(Shell.WINDOWS, "not windows")
+assert 0 == exec(["gawk", "--version"])
+ }
+
+ @Test
+ public void testHasXargsInstalled() throws Throwable {
+assume(Shell.WINDOWS, "not windows")
+assert 0 == exec(["xargs", "--version"])
+ }
+
+
@Test
public void testEmitKillCommand() throws Throwable {
killJavaProcesses("regionserver", 9)
}
@Test
+ public void testHadoopHomeDefined() throws Throwable {
+assume(Shell.WINDOWS, "not windows")
+def hadoopHome = Shell.hadoopHome
+log.info("HADOOP_HOME=$hadoopHome")
+ }
+
+ @Test
public void testHasWinutils() throws Throwable {
assume(Shell.WINDOWS, "not windows")
SliderUtils.verifyWinUtilsValid()
@@ -123,14 +144,27 @@ class TestWindowsSupport extends SliderTestBase {
assert winUtilsPath
File winUtils = new File(winUtilsPath)
log.debug("Winutils is at $winUtils)")
+
+ForkedProcessService process = exec([winUtilsPath, "systeminfo"])
+assert 0 == process.exitCode
+ }
+
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+ public ForkedProcessService exec(ArrayList commands) {
ForkedProcessService process;
process = new ForkedProcessService(
-methodName.methodName,
+methodName.methodName,
[:],
-[winUtilsPath, "systeminfo"]);
+commands);
process.init(new Configuration());
EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
process.start();
waiter.waitForServiceToStop(5000);
+process
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 58a76af..89f1a34 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | awk \"{print
\$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print
\$1}\" | xargs -n 1 taskkill /PID\""
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[26/47] git commit: Merge branch 'feature/SLIDER-394_env_check' into develop
Merge branch 'feature/SLIDER-394_env_check' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88d72ddf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88d72ddf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88d72ddf Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 88d72ddfb32977ff098b0329ae6f3faf2f7c2875 Parents: 6320f61 2d6f276 Author: Steve Loughran Authored: Thu Sep 4 17:12:02 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 17:12:02 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 18 +- .../common/params/AbstractActionArgs.java | 11 + .../slider/common/params/ActionHelpArgs.java| 14 +- .../slider/common/params/ActionVersionArgs.java | 12 +- .../apache/slider/common/tools/SliderUtils.java | 557 +-- .../slider/core/build/InstanceBuilder.java | 5 +- .../server/appmaster/SliderAppMaster.java | 5 +- .../services/utility/EndOfServiceWaiter.java| 84 +++ .../services/workflow/ForkedProcessService.java | 21 +- .../services/workflow/LongLivedProcess.java | 57 +- .../agent/AgentMiniClusterTestBase.groovy | 9 + .../tools/TestExecutionEnvironment.groovy | 42 ++ .../common/tools/TestWindowsSupport.groovy | 4 +- .../slider/providers/agent/AgentTestBase.groovy | 10 + .../services/workflow/EndOfServiceWaiter.java | 56 -- .../TestWorkflowForkedProcessService.java | 4 +- .../masterless/TestRoleOptPropagation.groovy| 4 +- 17 files changed, 663 insertions(+), 250 deletions(-) --
[14/47] git commit: SLIDER-350 add strict test for windows utils being valid
SLIDER-350 add strict test for windows utils being valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2e3c4c81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2e3c4c81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2e3c4c81
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2e3c4c81b799cbf6d06e394c2b17f8f161f68734
Parents: bfdec58
Author: Steve Loughran
Authored: Thu Sep 4 13:46:59 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 4 13:46:59 2014 +0100
--
.../apache/slider/common/tools/SliderUtils.java | 50 +++-
.../common/tools/TestWindowsSupport.groovy | 30 ++--
2 files changed, 76 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a864878..eb214db 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
@@ -63,6 +64,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -104,8 +106,9 @@ public final class SliderUtils {
public static final String JAVA_SECURITY_KRB5_REALM =
"java.security.krb5.realm";
public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+ public static final String WINUTILS = "WINUTILS.EXE";
+
-
private SliderUtils() {
}
@@ -1499,4 +1502,49 @@ public final class SliderUtils {
return is;
}
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void verifyWinUtilsValid() throws IOException {
+if (!Shell.WINDOWS) {
+ return;
+}
+String winUtilsPath = Shell.getWinUtilsPath();
+if (winUtilsPath == null) {
+ throw new FileNotFoundException(WINUTILS + " not found on Path : " +
+ System.getenv("Path"));
+}
+File winUtils = new File(winUtilsPath);
+if (!winUtils.isFile()) {
+ throw new FileNotFoundException(WINUTILS
+ + " at " + winUtilsPath
+ + " is not a file");
+
+}
+if (winUtils.length() < 0x100) {
+ throw new FileNotFoundException(WINUTILS
++ " at " + winUtilsPath
++ " is too short to be an executable");
+}
+// now read two bytes and verify the header.
+FileReader reader = null;
+try {
+ int[] header = new int[2];
+ reader = new FileReader(winUtilsPath);
+ header[0] = reader.read();
+ header[1] = reader.read();
+ if (header[0] != 'M' || header[1] != 'Z') {
+throw new FileNotFoundException(WINUTILS
+ + " at " + winUtilsPath
+ + " is not a windows executable file");
+ }
+} finally {
+ IOUtils.closeStream(reader);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 7f5edf3..84356ea 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -27,14 +27,16 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
import org.apache.slider.providers.agent.AgentUtils
-import o
[02/47] git commit: SLIDER-387 adjusted default accumulo configuration
SLIDER-387 adjusted default accumulo configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ac2536f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ac2536f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ac2536f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0ac2536f96e9010569deb0a0d2079f8c23c17e06
Parents: 7ecab23
Author: Billie Rinaldi
Authored: Tue Sep 2 11:17:46 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 2 11:17:46 2014 -0700
--
app-packages/accumulo/appConfig.json| 5 +++--
.../accumulo/src/test/resources/appConfig_monitor_ssl.json | 5 +++--
app-packages/accumulo/src/test/resources/appConfig_ssl.json | 5 +++--
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/appConfig.json
--
diff --git a/app-packages/accumulo/appConfig.json
b/app-packages/accumulo/appConfig.json
index a44dbad..2c0c2d5 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
-"site.global.tserver_heapsize": "128m",
+"site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,12 +27,13 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator":
"org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths":
"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+"site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
-"site.accumulo-site.tserver.walog.max.size": "100M",
+"site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
--
diff --git
a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
index c6a2639..45e6e19 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
-"site.global.tserver_heapsize": "128m",
+"site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,12 +27,13 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator":
"org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths":
"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+"site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
-"site.accumulo-site.tserver.walog.max.size": "100M",
+"site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_ssl.json
--
diff --git a/app-packages/accumulo/src/test/resources/appC
[45/47] git commit: Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f14d32c6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f14d32c6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f14d32c6 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f14d32c6fac2e252e587dbdc88e0e3e079f44d37 Parents: 2d6331a b323df8 Author: Steve Loughran Authored: Fri Sep 5 13:59:58 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 5 13:59:58 2014 +0100 -- pom.xml | 1 + slider-agent/pom.xml| 4 +-- slider-agent/src/main/python/setup.py | 2 +- slider-agent/src/test/python/python-wrap| 40 -- slider-agent/src/test/python/python-wrap.sh | 42 slider-agent/src/test/python/unitTests.py | 36 ++-- slider-core/pom.xml | 4 +-- 7 files changed, 65 insertions(+), 64 deletions(-) --
[33/47] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/930920d2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/930920d2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/930920d2 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 930920d245bb354cbfbec43391a203bd6db04fa4 Parents: b7f37ea 6126db6 Author: Steve Loughran Authored: Thu Sep 4 19:00:55 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 19:00:55 2014 +0100 -- pom.xml | 41 +++-- 1 file changed, 39 insertions(+), 2 deletions(-) --
[37/47] git commit: Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e2a80326 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e2a80326 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e2a80326 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: e2a80326a7f7ddc6f2bc8b00dbaa5265726a6bb0 Parents: 930920d ee3ad77 Author: Steve Loughran Authored: Thu Sep 4 21:02:30 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 21:02:30 2014 +0100 -- .../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +- .../test/groovy/org/apache/slider/test/SliderTestUtils.groovy| 4 ++-- .../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) --
[44/47] git commit: SLIDER-401 agent tests on windows
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b323df81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b323df81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b323df81
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b323df811a138a0880b60be95d9150419d1dcb85
Parents: ce3f700
Author: Steve Loughran
Authored: Fri Sep 5 13:53:05 2014 +0100
Committer: Steve Loughran
Committed: Fri Sep 5 13:53:05 2014 +0100
--
slider-agent/src/test/python/unitTests.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b323df81/slider-agent/src/test/python/unitTests.py
--
diff --git a/slider-agent/src/test/python/unitTests.py
b/slider-agent/src/test/python/unitTests.py
index bab121f..6aa0167 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -70,8 +70,9 @@ def all_tests_suite():
else:
for file_name in files_list:
if file_name.endswith(PY_EXT) and not file_name == __file__:
-logger.info(file_name)
-tests_list.append(file_name.replace(PY_EXT, ''))
+replaced = file_name.replace(PY_EXT, '')
+logger.info(replaced)
+tests_list.append(replaced)
logger.info('')
suite = unittest.TestLoader().loadTestsFromNames(tests_list)
[19/47] git commit: Merge branch 'feature/SLIDER-377-windows' into develop
Merge branch 'feature/SLIDER-377-windows' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29a40f50 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29a40f50 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29a40f50 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 29a40f5094b4275fa4d7159d91f1c965cdc6da50 Parents: 38c816e 877a016 Author: Steve Loughran Authored: Thu Sep 4 14:41:54 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 4 14:41:54 2014 +0100 -- .../apache/slider/common/tools/SliderUtils.java | 50 - .../common/tools/TestWindowsSupport.groovy | 76 +++- .../apache/slider/test/SliderTestUtils.groovy | 2 +- 3 files changed, 123 insertions(+), 5 deletions(-) --
[27/47] git commit: SLIDER-377 tagging some test files as @CompileStatic
SLIDER-377 tagging some test files as @CompileStatic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2965d8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2965d8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2965d8a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f2965d8a056fcf6d56a39f0841f1903048c95c29
Parents: 88d72dd
Author: Steve Loughran
Authored: Thu Sep 4 17:54:46 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 4 17:54:46 2014 +0100
--
.../slider/common/tools/TestClientResourceRegistration.groovy | 3 +++
.../org/apache/slider/common/tools/TestConfigHelperHDFS.groovy| 3 +++
.../org/apache/slider/common/tools/TestMiscSliderUtils.groovy | 3 +++
3 files changed, 9 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
index 7150b3c..e3bfa1b 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestClientResourceRegistration {
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
index 0d21d6c..d78aa1b 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
@@ -27,6 +28,8 @@ import org.apache.slider.test.YarnMiniClusterTestBase
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
//diabled for now;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
index 24367a3..9042af5 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.slider.test.SliderTestBase
import org.junit.Test
+@CompileStatic
+
class TestMiscSliderUtils extends SliderTestBase {
[10/47] git commit: SLIDER-181. Storm package updates
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38c816eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38c816eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38c816eb
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 38c816ebbd3c883308bbe36fc00846d4ff4dac7b
Parents: e9a353c
Author: Sumit Mohanty
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Sep 3 13:46:02 2014 -0700
--
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16
app-packages/storm/pom.xml | 90
app-packages/storm/src/assembly/storm.xml | 68 +++
6 files changed, 172 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/README.txt
--
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific
location.
-Various configurations provided in this sample are customized for
apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn
command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=
+ -Dpkg.name= -Dpkg.src=
-Create a zip package at the root of the package (/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz
-Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they
work
well as the default configuration for Slider apps. So its advisable that when
you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
-"application.def": "apache-storm-0.9.3.zip",
+"application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
-"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
-"site.storm-site.worker.childopts": "-Xmx768m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireform
[03/47] git commit: SLIDER-388 allow default zookeeper port in ZookeeperUtils
SLIDER-388 allow default zookeeper port in ZookeeperUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b525287d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b525287d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b525287d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b525287dd05a29af6308890aef44d524a6341296
Parents: 0ac2536
Author: Billie Rinaldi
Authored: Tue Sep 2 12:39:06 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 2 12:39:06 2014 -0700
--
.../src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b525287d/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 61b1ff0..cc1b2c9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.List;
public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
public static String buildConnectionString(String zkHosts, int port) {
String zkPort = Integer.toString(port);
@@ -73,7 +74,7 @@ public class ZookeeperUtils {
List list = new ArrayList(len);
if (strings != null) {
for (String s : strings) {
-list.add(HostAndPort.fromString(s.trim()));
+
list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
}
}
return list;
[17/47] git commit: SLIDER-350 windows support
SLIDER-350 windows support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93c3c770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93c3c770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93c3c770
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 93c3c770fed878895e4fb67ac17cbbb098cb4fb3
Parents: 8f9caa9
Author: Steve Loughran
Authored: Thu Sep 4 14:05:38 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 4 14:05:38 2014 +0100
--
.../common/tools/TestWindowsSupport.groovy | 22 +++-
1 file changed, 17 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/93c3c770/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 61ede62..6ae683a 100644
---
a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -109,13 +109,13 @@ class TestWindowsSupport extends SliderTestBase {
@Test
public void testHasGawkInstalled() throws Throwable {
assume(Shell.WINDOWS, "not windows")
-assert 0 == exec(["gawk", "--version"])
+exec(0, ["gawk", "--version"])
}
@Test
public void testHasXargsInstalled() throws Throwable {
assume(Shell.WINDOWS, "not windows")
-assert 0 == exec(["xargs", "--version"])
+exec(0, ["xargs", "--version"])
}
@@ -145,16 +145,28 @@ class TestWindowsSupport extends SliderTestBase {
File winUtils = new File(winUtilsPath)
log.debug("Winutils is at $winUtils)")
-ForkedProcessService process = exec([winUtilsPath, "systeminfo"])
-assert 0 == process.exitCode
+exec(0, [winUtilsPath, "systeminfo"])
}
/**
* Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
* @param commands
- * @return
+ * @return the process
*/
+ public ForkedProcessService exec(int status, List commands) {
+ForkedProcessService process = exec(commands)
+assert status == process.exitCode
+return process
+ }
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+
public ForkedProcessService exec(List commands) {
ForkedProcessService process;
process = new ForkedProcessService(
[43/47] git commit: SLIDER-401 agent tests on windows
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce3f7009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce3f7009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce3f7009
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ce3f7009e6e9fec204b264a9577a52632b9f3487
Parents: a9f0b98
Author: Steve Loughran
Authored: Fri Sep 5 13:47:50 2014 +0100
Committer: Steve Loughran
Committed: Fri Sep 5 13:47:50 2014 +0100
--
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/unitTests.py | 31 --
2 files changed, 15 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/main/python/setup.py
--
diff --git a/slider-agent/src/main/python/setup.py
b/slider-agent/src/main/python/setup.py
index 421b5f9..56969b6 100644
--- a/slider-agent/src/main/python/setup.py
+++ b/slider-agent/src/main/python/setup.py
@@ -17,7 +17,7 @@ from setuptools import setup
setup(
name = "slider-agent",
-version = "0.31.0-incubating-SNAPSHOT",
+version = "0.51.0-incubating-SNAPSHOT",
packages = ['agent'],
# metadata for upload to PyPI
author = "Apache Software Foundation",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/test/python/unitTests.py
--
diff --git a/slider-agent/src/test/python/unitTests.py
b/slider-agent/src/test/python/unitTests.py
index b01438e..bab121f 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,14 +16,13 @@ distributed under the License is distributed on an "AS IS"
BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
import unittest
-import doctest
-from os.path import dirname, split, isdir
-import logging.handlers
+from os.path import isdir
import logging
-from random import shuffle
+import os
+import sys
LOG_FILE_NAME='tests.log'
SELECTED_PREFIX = "_"
@@ -31,9 +30,9 @@ PY_EXT='.py'
ignoredDirs = ["mock"]
class TestAgent(unittest.TestSuite):
- def run(self, result):
+ def run(self, result, debug=False):
run = unittest.TestSuite.run
-run(self, result)
+run(self, result, debug)
return result
@@ -41,19 +40,19 @@ def parent_dir(path):
if isdir(path):
if path.endswith(os.sep):
path = os.path.dirname(path)
-parent_dir = os.path.dirname(path)
+parent = os.path.dirname(path)
else:
-parent_dir = os.path.dirname(os.path.dirname(path))
+parent = os.path.dirname(os.path.dirname(path))
- return parent_dir
+ return parent
def all_tests_suite():
- src_dir = os.getcwd()
+ root_dir = os.getcwd()
files_list = []
- for directory in os.listdir(src_dir):
+ for directory in os.listdir(root_dir):
if os.path.isdir(directory) and not directory in ignoredDirs:
- files_list += os.listdir(src_dir + os.sep + directory)
+ files_list += os.listdir(root_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
files_list.sort()
@@ -99,9 +98,7 @@ def main():
logger.info('')
if __name__ == '__main__':
- import os
- import sys
- import io
+
sys.path.insert(0,
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)
sys.path.insert(0,
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__ +
os.sep + 'main' + os.sep + 'python')
sys.path.insert(0,
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__ +
os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
[35/47] git commit: SLIDER-397 taskkill /f
SLIDER-397 taskkill /f
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27f0a625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27f0a625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27f0a625
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 27f0a625265de8e0d818917c94b2f0d5385b9044
Parents: 132d9f2
Author: Steve Loughran
Authored: Thu Sep 4 20:39:05 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 4 20:39:05 2014 +0100
--
.../test/groovy/org/apache/slider/test/SliderTestUtils.groovy| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27f0a625/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 89f1a34..5086248 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -747,7 +747,7 @@ class SliderTestUtils extends Assert {
def commandString
if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill $signal"
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' |
xargs kill /f $signal"
log.info("Command command = $killCommand" )
commandString = ["bash", "-c", killCommand]
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print
\$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print
\$1}\" | xargs -n 1 taskkill /f /PID "
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[20/47] git commit: SLIDER-146 added sample appConfig that enables kerberos for Accumulo
SLIDER-146 added sample appConfig that enables kerberos for Accumulo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6320f618
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6320f618
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6320f618
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6320f6186dec56f68afbbd638f5ddc0e7dc04bff
Parents: 29a40f5
Author: Billie Rinaldi
Authored: Thu Sep 4 07:15:22 2014 -0700
Committer: Billie Rinaldi
Committed: Thu Sep 4 07:15:22 2014 -0700
--
.../src/test/resources/appConfig_kerberos.json | 55
1 file changed, 55 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6320f618/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
--
diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
new file mode 100644
index 000..c9684d4
--- /dev/null
+++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
@@ -0,0 +1,55 @@
+{
+ "schema": "http://example.org/specification/v2.0.0";,
+ "metadata": {
+ },
+ "global": {
+"application.def": "${app.package.name}.zip",
+"java_home": "/usr/lib/jvm/java",
+"site.global.app_user": "accumulo",
+"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
+"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+"site.global.tserver_heapsize": "256m",
+"site.global.master_heapsize": "128m",
+"site.global.monitor_heapsize": "64m",
+"site.global.gc_heapsize": "64m",
+"site.global.other_heapsize": "128m",
+"site.global.hadoop_prefix": "/usr/lib/hadoop",
+"site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+"site.global.zookeeper_home": "/usr/lib/zookeeper",
+"site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
+"site.global.accumulo_root_password": "NOT_USED",
+"site.global.user_group": "accumulo",
+"site.global.security_enabled": "false",
+"site.global.ssl_cert_dir": "ssl",
+"site.global.monitor_protocol": "http",
+"site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+"site.accumulo-site.instance.security.authenticator":
"org.apache.slider.accumulo.CustomAuthenticator",
+"site.accumulo-site.general.security.credential.provider.paths":
"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+"site.accumulo-site.general.kerberos.keytab":
"/etc/security/keytabs/accumulo.service.keytab",
+"site.accumulo-site.general.kerberos.principal":
"accumulo/_h...@example.com",
+"site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+"site.accumulo-site.tserver.memory.maps.max": "80M",
+"site.accumulo-site.tserver.cache.data.size": "7M",
+"site.accumulo-site.tserver.cache.index.size": "20M",
+"site.accumulo-site.trace.user": "root",
+"site.accumulo-site.tserver.sort.buffer.size": "50M",
+"site.accumulo-site.tserver.walog.max.size": "40M",
+"site.accumulo-site.master.port.client": "0",
+"site.accumulo-site.trace.port.client": "0",
+"site.accumulo-site.tserver.port.client": "0",
+"site.accumulo-site.gc.port.client": "0",
+"site.accumulo-site.monitor.port.client":
"${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+"site.accumulo-site.monitor.port.log4j": "0",
+"site.accumulo-site.general.classpaths":
"$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
+ },
+ "credentials": {
+"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks":
["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+"slider-appmaster": {
+ "jvm.heapsize": "256M"
+}
+ }
+}
[11/47] git commit: Merge branch 'develop' into feature/auto_generate_storm_pkg
Merge branch 'develop' into feature/auto_generate_storm_pkg Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26b4153e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26b4153e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26b4153e Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 26b4153ebf070521fb5030d57846f4ccd7f45cb0 Parents: e1129b0 38c816e Author: Sumit Mohanty Authored: Wed Sep 3 13:46:44 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 3 13:46:44 2014 -0700 -- .../main/java/org/apache/slider/api/InternalKeys.java | 11 +++ .../main/java/org/apache/slider/api/ResourceKeys.java | 13 - .../apache/slider/server/appmaster/state/AppState.java | 9 + 3 files changed, 16 insertions(+), 17 deletions(-) --
[07/47] git commit: SLIDER-390 interpret short lived threshold as seconds instead of milliseconds
SLIDER-390 interpret short lived threshold as seconds instead of milliseconds
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0b7bd38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0b7bd38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0b7bd38
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b0b7bd38542ed287415066d0215c6fda8786336e
Parents: 201686e
Author: Billie Rinaldi
Authored: Wed Sep 3 08:38:52 2014 -0700
Committer: Billie Rinaldi
Committed: Wed Sep 3 08:38:52 2014 -0700
--
.../java/org/apache/slider/server/appmaster/state/AppState.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0b7bd38/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 8b70185..dce48d1 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1227,7 +1227,7 @@ public class AppState {
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
- * @return true if the instance is considered short live
+ * @return true if the instance is considered short lived
*/
@VisibleForTesting
public boolean isShortLived(RoleInstance instance) {
@@ -1236,7 +1236,7 @@ public class AppState {
boolean shortlived;
if (started > 0) {
long duration = time - started;
- shortlived = duration < startTimeThreshold;
+ shortlived = duration < (startTimeThreshold * 1000);
} else {
// never even saw a start event
shortlived = true;
git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows [created] be57d7a11
SLIDER-414 RAT is creating a dir that cannot be deleted
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1
Branch: refs/heads/feature/SLIDER-86-windows
Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef
Parents: 65d17d8
Author: Steve Loughran
Authored: Tue Sep 9 18:26:52 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 18:26:52 2014 +0100
--
.../command-logger/application-pkg/pom.xml | 38 ---
app-packages/command-logger/slider-pkg/pom.xml | 38 ---
pom.xml | 67 ++--
slider-agent/pom.xml| 62 ++-
slider-core/pom.xml | 102 +--
slider-funtest/pom.xml | 48 +
.../hbase/slider-hbase-provider/pom.xml | 49 +
src/test/clusters/script.md | 2 +-
8 files changed, 208 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml
--
diff --git a/app-packages/command-logger/application-pkg/pom.xml
b/app-packages/command-logger/application-pkg/pom.xml
index 77f0fc9..051ac84 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -54,20 +54,8 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
+
org.apache.maven.wagon
@@ -75,4 +63,28 @@
+
+
+
+ rat
+
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml
--
diff --git a/app-packages/command-logger/slider-pkg/pom.xml
b/app-packages/command-logger/slider-pkg/pom.xml
index b9bbf06..4550ae1 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -93,20 +93,6 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
@@ -118,5 +104,29 @@
tar
+
+
+
+ apache-release
+
+
+
+
+rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 167b9ef..df8599f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,36 +349,7 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
-**/*.json
-**/*.tar
-**/build.properties
-**/regionservers
-**/slaves
-**/httpfs-signature.secret
-**/dfs.exclude
-**/*.iml
-**/rat.txt
-DISCLAIMER
-app-packages/hbase/target/**
-target/*
-
-
-
-
+
@@ -1263,6 +1234,42 @@
+ rat
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+**
git commit: SLIDER-86 failsafe not passing down sysprops
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows be57d7a11 -> 2cdb1ba00
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:23:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:23:54 2014 +0100
--
slider-funtest/pom.xml | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml
--
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 124251f..2b55359 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -109,6 +109,12 @@
false
${build.redirect.test.output.to.file}
+
+
+ slider.bin.dir
+
../slider-assembly/target/slider-${project.version}-all/slider-${project.version}
+
+
true
true
[1/2] git commit: SLIDER-86 failsafe not passing down sysprops
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows 2cdb1ba00 -> 8f6f15ff6
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:43:39 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:43:39 2014 +0100
--
.../slider/funtest/basic/SyspropsIT.groovy | 50
1 file changed, 50 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
--
diff --git
a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
new file mode 100644
index 000..728920e
--- /dev/null
+++
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on CommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@Slf4j
+class SyspropsIT extends SliderTestUtils{
+
+
+ @Test
+ public void testDumpSysprops() throws Throwable {
+def sysprops = System.properties
+TreeSet sorted = new TreeSet();
+sysprops.keys().each { String it -> sorted.add(it)}
+sorted.each { String key ->
+ log.info("$key=\"${sysprops[key]}\"")
+}
+
+ }
+}
[2/2] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/feature/SLIDER-86-windows Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[3/3] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17
Branch: refs/heads/feature/SLIDER-86-windows
Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4
Parents: d6e73cd
Author: Steve Loughran
Authored: Wed Sep 10 12:13:22 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:13:22 2014 +0100
--
.../main/java/org/apache/slider/client/SliderClient.java | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8d0978c..606a53c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -201,11 +201,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
addService(yarnClient);
super.serviceInit(conf);
-
-//here the superclass is inited; getConfig returns a non-null value
-sliderFileSystem = new SliderFileSystem(getConfig());
-YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+
}
/**
@@ -298,6 +294,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
if (coreAction.getHadoopServicesRequired()) {
// validate the client
SliderUtils.validateSliderClientEnvironment(null);
+
+ //here the superclass is inited; getConfig returns a non-null value
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ YARNRegistryClient =
+ new YARNRegistryClient(yarnClient, getUsername(), getConfig());
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
[2/3] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8
Branch: refs/heads/feature/SLIDER-86-windows
Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea
Parents: 1efc358
Author: Steve Loughran
Authored: Wed Sep 10 12:12:42 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:42 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 107
1 file changed, 74 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index cbc0faf..4b078e7 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -18,6 +18,8 @@
import sys
import os
import subprocess
+import time
+from threading import Thread
CONF = "conf"
@@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m -Djava.confdir=%s"
+ON_POSIX = 'posix' in sys.builtin_module_names
+
+finished = False
+DEBUG = True
+
"""
Launches slider
+Nonblocking IO on windows is "tricky" ... see
+http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+to explain the code here
+
"""
@@ -66,6 +77,20 @@ def dirMustExist(dirname):
raise Exception("Directory does not exist: %s " % dirname)
return dirname
+
+def debug(text):
+ if DEBUG: print '[DEBUG] ' + text
+
+
+def error(text):
+ print '[ERROR] ' + text
+ sys.stdout.flush()
+
+def info(text):
+ print text
+ sys.stdout.flush()
+
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -86,55 +111,71 @@ def read(pipe, line):
return line, False
+def print_output(name, src):
+ """
+ Relay the output stream to stdout line by line
+ :param name:
+ :param src: source stream
+ :return:
+ """
+
+ debug ("starting printer for %s" % name )
+ line = ""
+ while not finished:
+(line, done) = read(src, line)
+if done:
+ info(name +': ' + line)
+ line = ""
+ src.close()
+
+
def runProcess(commandline):
"""
Run a process
:param commandline: command line
:return:the return code
"""
- print "ready to exec : %s" % commandline
+ debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
+ shell=False,
+ bufsize=1,
+ close_fds=ON_POSIX)
+
+ t = Thread(target=print_output, args=("stdout", exe.stdout))
+ t.daemon = True
+ t.start()
+ t2 = Thread(target=print_output, args=("stderr", exe.stderr,))
+ t2.daemon = True
+ t2.start()
+
+ debug("Waiting for completion")
while exe.poll() is None:
# process is running; grab output and echo every line
-outline, done = read(stdout, outline)
-if done:
- print outline
- outline = ""
-errline, done = read(stderr, errline)
-if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
+time.sleep(1)
+ debug("completed with exit code : %d" % exe.returncode)
return exe.returncode
-def which(program):
-def is_exe(fpath):
-return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-fpath, fname = os.path.split(program)
-if fpath:
-if is_exe(program):
-return program
-else:
-for path in os.environ["PATH"].split(os.pathsep):
-path = path.strip('"')
-exe_file = os.path.join(path, program)
-if is_exe(exe_file):
-return exe_file
+def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+def which(program):
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+if is_exe(program):
+ return program
+ else:
+for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+return ex
[1/3] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows 8f6f15ff6 -> a6526a17f
SLIDER-86: slider funtests call cmd /c slider.py on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc
Parents: 8f6f15f
Author: Steve Loughran
Authored: Wed Sep 10 12:12:10 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:10 2014 +0100
--
.../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++-
.../apache/slider/funtest/framework/FuntestProperties.groovy | 1 +
.../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 3b52912..b715e38 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
@@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final File SLIDER_SCRIPT = new File(
SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
+ public static final File SLIDER_SCRIPT_PYTHON = new File(
+ SLIDER_TAR_DIRECTORY,
+ BIN_SLIDER_PYTHON).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
@@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = SLIDER_SCRIPT
+SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 1096dfa..730639b 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,6 +57,7 @@ public interface FuntestProperties extends
SliderXMLConfKeysForTesting {
String KEY_TEST_CONF_XML = "slider.test.conf.xml"
String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
String BIN_SLIDER = "bin/slider"
+ String BIN_SLIDER_PYTHON = "bin/slider.py"
String AGENT_INI = "agent.ini"
String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 804e791..3d856c7 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -30,6 +30,7 @@ class SliderShell extends Shell {
public static final String BASH = '/bin/bash -s'
+ public static final String CMD = 'cmd /c'
/**
* Configuration directory, shared across all inst
[2/2] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3
Branch: refs/heads/feature/SLIDER-86-windows
Commit: f7906fa3a6a5fe04113832b747cbe378e5949219
Parents: d51ce5a
Author: Steve Loughran
Authored: Wed Sep 10 12:25:41 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:41 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index 4b078e7..735e09b 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -135,6 +135,7 @@ def runProcess(commandline):
:param commandline: command line
:return:the return code
"""
+ global finished
debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
@@ -156,6 +157,7 @@ def runProcess(commandline):
# process is running; grab output and echo every line
time.sleep(1)
debug("completed with exit code : %d" % exe.returncode)
+ finished = True
return exe.returncode
[1/2] git commit: SLIDER-418 sider version command fails if the config isn't valid
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows a6526a17f -> f7906fa3a
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1
Branch: refs/heads/feature/SLIDER-86-windows
Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f
Parents: a6526a1
Author: Steve Loughran
Authored: Wed Sep 10 12:25:32 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:32 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 37 ++--
1 file changed, 27 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 606a53c..ef0e63c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -196,9 +196,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-//create the YARN client
-yarnClient = new SliderYarnClientImpl();
-addService(yarnClient);
+
super.serviceInit(conf);
@@ -292,13 +290,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
- // validate the client
- SliderUtils.validateSliderClientEnvironment(null);
-
- //here the superclass is inited; getConfig returns a non-null value
- sliderFileSystem = new SliderFileSystem(getConfig());
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ initHadoopBinding();
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
@@ -353,6 +345,31 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+
+ /**
+ * Perform everything needed to init the hadoop binding.
+ * This assumes that the service is already in inited or started state
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected void initHadoopBinding() throws IOException, SliderException {
+// validate the client
+SliderUtils.validateSliderClientEnvironment(null);
+//create the YARN client
+yarnClient = new SliderYarnClientImpl();
+yarnClient.init(getConfig());
+if (getServiceState() == STATE.STARTED) {
+ yarnClient.start();
+}
+addService(yarnClient);
+// create the filesystem
+sliderFileSystem = new SliderFileSystem(getConfig());
+
+// and the registry
+YARNRegistryClient =
+new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ }
+
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows f7906fa3a -> f2270e34b
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34
Branch: refs/heads/feature/SLIDER-86-windows
Commit: f2270e34b610f7e7754bc51226f66d30ca55c581
Parents: f7906fa
Author: Steve Loughran
Authored: Wed Sep 10 13:01:14 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:01:14 2014 +0100
--
.../funtest/framework/CommandTestBase.groovy| 2 +-
.../slider/funtest/framework/SliderShell.groovy | 84 ++--
2 files changed, 77 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index b715e38..216b68d 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON :
SLIDER_SCRIPT
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 3d856c7..e49aee4 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory
class SliderShell extends Shell {
private static final Logger log = LoggerFactory.getLogger(SliderShell.class);
-
+ private static final Logger LOG = log;
public static final String BASH = '/bin/bash -s'
- public static final String CMD = 'cmd /c'
+ public static final String CMD = 'cmd'
/**
* Configuration directory, shared across all instances. Not marked as
volatile,
@@ -38,7 +38,7 @@ class SliderShell extends Shell {
*/
public static File confDir;
- public static File script;
+ public static File scriptFile;
public static final List slider_classpath_extra = []
@@ -51,8 +51,8 @@ class SliderShell extends Shell {
SliderShell(Collection commands) {
super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH)
assert confDir != null;
-assert script != null;
-command = script.absolutePath + " " + commands.join(" ")
+assert scriptFile != null;
+command = scriptFile.absolutePath + " " + commands.join(" ")
}
/**
@@ -67,7 +67,9 @@ class SliderShell extends Shell {
]
if (!slider_classpath_extra.empty) {
commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
- SliderUtils.join(slider_classpath_extra, ":", false))
+ SliderUtils.join(slider_classpath_extra,
+ (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"),
+ false))
}
commandLine << command
String script = commandLine.join("\n")
@@ -77,8 +79,18 @@ class SliderShell extends Shell {
return ret;
}
+ /**
+ * Add an environment variable
+ * @param var variable
+ * @param val value (which will be stringified)
+ * @return an env variable command
+ */
String env(String var, Object val) {
-return "export " + var + "=${val.toString()};"
+if (org.apache.hadoop.util.Shell.WINDOWS) {
+ return "set " + var + "=${val.toString()};"
+} else {
+ return "export " + var + "=${val.toString()};"
+}
}
/**
@@ -152,6 +164,62 @@ class SliderShell extends Shell {
}
}
-
+ /**
+ * Execute shell script consisting of as many Strings as we
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows f2270e34b -> 49ae17310
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b
Parents: f2270e3
Author: Steve Loughran
Authored: Wed Sep 10 13:05:25 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:05:25 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index e49aee4..6effb46 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -87,7 +87,7 @@ class SliderShell extends Shell {
*/
String env(String var, Object val) {
if (org.apache.hadoop.util.Shell.WINDOWS) {
- return "set " + var + "=${val.toString()};"
+ return "set " + var + "=${val.toString()}"
} else {
return "export " + var + "=${val.toString()};"
}
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows 49ae17310 -> 682ae8a7f
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 682ae8a7fa92932f32420486761efa06b5febbc1
Parents: 49ae173
Author: Steve Loughran
Authored: Wed Sep 10 13:40:17 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:40:17 2014 +0100
--
.../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 50f3ac0..81d1595 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
log.info "App pkg uploaded at $appPkgPath"
} catch (Exception e) {
setup_failed = true
- fail("Setup failed "+e)
+ throw e;
}
}
git commit: SLIDER-420 slider funtests not working on windows
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-86-windows 682ae8a7f -> 16d625e48
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 16d625e4800d2a64b22f62810a832120c09869f3
Parents: 682ae8a
Author: Steve Loughran
Authored: Wed Sep 10 13:45:35 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:45:35 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 921adbf..70f119e 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -145,6 +145,7 @@ class FileUploader {
public void attemptToCreateHomeDir(String username, Path home) {
def privilegedFS = getFileSystemAsUserName(username)
+log.info "Creating home dir $home as user ${user.userName} group
${user.primaryGroupName}"
privilegedFS.mkdirs(home, new FsPermission((short) 00755))
privilegedFS.setOwner(home, user.userName, user.primaryGroupName)
}
[15/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731
Branch: refs/heads/develop
Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b
Parents: f2270e3
Author: Steve Loughran
Authored: Wed Sep 10 13:05:25 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:05:25 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index e49aee4..6effb46 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -87,7 +87,7 @@ class SliderShell extends Shell {
*/
String env(String var, Object val) {
if (org.apache.hadoop.util.Shell.WINDOWS) {
- return "set " + var + "=${val.toString()};"
+ return "set " + var + "=${val.toString()}"
} else {
return "export " + var + "=${val.toString()};"
}
[16/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7
Branch: refs/heads/develop
Commit: 682ae8a7fa92932f32420486761efa06b5febbc1
Parents: 49ae173
Author: Steve Loughran
Authored: Wed Sep 10 13:40:17 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:40:17 2014 +0100
--
.../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 50f3ac0..81d1595 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
log.info "App pkg uploaded at $appPkgPath"
} catch (Exception e) {
setup_failed = true
- fail("Setup failed "+e)
+ throw e;
}
}
[13/19] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3
Branch: refs/heads/develop
Commit: f7906fa3a6a5fe04113832b747cbe378e5949219
Parents: d51ce5a
Author: Steve Loughran
Authored: Wed Sep 10 12:25:41 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:41 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index 4b078e7..735e09b 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -135,6 +135,7 @@ def runProcess(commandline):
:param commandline: command line
:return:the return code
"""
+ global finished
debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
@@ -156,6 +157,7 @@ def runProcess(commandline):
# process is running; grab output and echo every line
time.sleep(1)
debug("completed with exit code : %d" % exe.returncode)
+ finished = True
return exe.returncode
[08/19] git commit: SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode
SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to
work in secure mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c1d9dc2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c1d9dc2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c1d9dc2
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 3c1d9dc2fcec1a897a26388865194c99fc28e2f8
Parents: e24060b
Author: Billie Rinaldi
Authored: Tue Sep 9 22:14:36 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 9 22:14:36 2014 -0700
--
app-packages/accumulo/README.txt| 17 ++--
app-packages/accumulo/pom.xml | 90 ++--
app-packages/accumulo/src/assembly/accumulo.xml | 2 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++
.../accumulo/AccumuloMonitorSSLIT.groovy| 7 +-
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +-
.../accumulo/src/test/resources/appConfig.json | 55
.../src/test/resources/appConfig_kerberos.json | 8 +-
.../test/resources/appConfig_monitor_ssl.json | 4 +-
.../appConfig_monitor_ssl_kerberos.json | 57 +
.../src/test/resources/appConfig_ssl.json | 4 +-
.../test/resources/appConfig_ssl_kerberos.json | 59 +
12 files changed, 294 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/README.txt
--
diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt
index 634296a..f75814c 100644
--- a/app-packages/accumulo/README.txt
+++ b/app-packages/accumulo/README.txt
@@ -17,21 +17,28 @@
# How to create a Slider package for Accumulo?
- mvn clean package -DskipTests -Paccumulo-app-package
+ mvn clean package -DskipTests -Paccumulo-app-package-maven
+ OR
+ mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \
+-Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball
App package can be found in
-
app-packages/accumulo/target/apache-slider-accumulo-${accumulo.version}-app-package-${slider.version}.zip
+ app-packages/accumulo/target/slider-accumulo-app-package-*.zip
+
+In the first case, the version number of the app package will match the
+slider version, and in the second case it will match the pkg.version
+(intended to be the accumulo version).
Verify the content using
- zip -Tv apache-slider-accumulo-*.zip
+ zip -Tv slider-accumulo-app-package*.zip
While appConfig.json and resources.json are not required for the package they
work well as the default configuration for Slider apps. So it is advisable that
when you create an application package for Slider, include sample/default
resources.json and appConfig.json for a minimal Yarn cluster.
-The version of Accumulo used for the app package can be adjusted by adding a
-flag such as
+In the maven packaging case, the version of Accumulo used for the app package
+can be adjusted by adding a flag such as
-Daccumulo.version=1.5.1
**Note that the LICENSE.txt and NOTICE.txt that are bundled with the app
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 548ced8..f074b01 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -30,17 +30,20 @@
package-tmp
-
${project.artifactId}-${project.version}
+${project.artifactId}-${pkg.version}
+${project.build.directory}/${work.dir}
+${project.version}
+accumulo-${accumulo.version}-bin.tar.gz
+
+yarn
+hadoop
+
+
accumulo-app-package
-
-
- !skipAccumulo
-
-
@@ -64,6 +67,79 @@
org.apache.maven.plugins
+maven-antrun-plugin
+1.7
+
+
+copy
+validate
+
+
+
+
+
+
+ run
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-failsafe-plugin
+${maven-failsafe-plugin.version}
+
+
+run-integration-tests
+
+ integration-test
+ verify
+
+
+
+
+
+true
+true
+
[07/19] git commit: SLIDER-416 substitute hostname for _HOST in principals
SLIDER-416 substitute hostname for _HOST in principals
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e24060b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e24060b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e24060b9
Branch: refs/heads/feature/SLIDER-86-windows
Commit: e24060b93751f9a73592be0ca628d3a403d044a3
Parents: ce0c052
Author: Billie Rinaldi
Authored: Tue Sep 9 20:17:30 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 9 20:17:30 2014 -0700
--
.../java/org/apache/slider/core/launch/AppMasterLauncher.java | 6 +-
.../server/services/security/FsDelegationTokenManager.java | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index bd8a0a5..c9ac20c 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -20,6 +20,7 @@ package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -35,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
@@ -196,7 +198,9 @@ public class AppMasterLauncher extends AbstractLauncher {
*/
private void addSecurityTokens() throws IOException {
-String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL);
+String tokenRenewer = SecurityUtil.getServerPrincipal(
+getConf().get(YarnConfiguration.RM_PRINCIPAL),
+InetAddress.getLocalHost().getCanonicalHostName());
if (SliderUtils.isUnset(tokenRenewer)) {
throw new IOException(
"Can't get Master Kerberos principal for the RM to use as renewer: "
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index cce2de0..c892b10 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -35,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
import java.security.PrivilegedExceptionAction;
import java.text.DateFormat;
import java.util.Date;
@@ -67,7 +69,9 @@ public class FsDelegationTokenManager {
// using HDFS principal...
this.remoteUser = UserGroupInformation
.loginUserFromKeytabAndReturnUGI(
-loginConfig.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
+SecurityUtil.getServerPrincipal(
+
loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
+InetAddress.getLocalHost().getCanonicalHostName()),
loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
log.info("Created remote user {}. UGI reports current user is {}",
this.remoteUser, UserGroupInformation.getCurrentUser());
[19/19] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/develop Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[09/19] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
SLIDER-86: slider funtests call cmd /c slider.py on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583
Branch: refs/heads/develop
Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc
Parents: 8f6f15f
Author: Steve Loughran
Authored: Wed Sep 10 12:12:10 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:10 2014 +0100
--
.../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++-
.../apache/slider/funtest/framework/FuntestProperties.groovy | 1 +
.../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 3b52912..b715e38 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
@@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final File SLIDER_SCRIPT = new File(
SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
+ public static final File SLIDER_SCRIPT_PYTHON = new File(
+ SLIDER_TAR_DIRECTORY,
+ BIN_SLIDER_PYTHON).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
@@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = SLIDER_SCRIPT
+SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 1096dfa..730639b 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,6 +57,7 @@ public interface FuntestProperties extends
SliderXMLConfKeysForTesting {
String KEY_TEST_CONF_XML = "slider.test.conf.xml"
String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
String BIN_SLIDER = "bin/slider"
+ String BIN_SLIDER_PYTHON = "bin/slider.py"
String AGENT_INI = "agent.ini"
String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 804e791..3d856c7 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -30,6 +30,7 @@ class SliderShell extends Shell {
public static final String BASH = '/bin/bash -s'
+ public static final String CMD = 'cmd /c'
/**
* Configuration directory, shared across all instances. Not marked as
volatile,
@@ -48,7 +49,7 @@ class SliderShell extends Shell {
* @param commands
*/
SliderShell(C
[04/19] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b
Branch: refs/heads/develop
Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:43:39 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:43:39 2014 +0100
--
.../slider/funtest/basic/SyspropsIT.groovy | 50
1 file changed, 50 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
--
diff --git
a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
new file mode 100644
index 000..728920e
--- /dev/null
+++
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on CommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@Slf4j
+class SyspropsIT extends SliderTestUtils{
+
+
+ @Test
+ public void testDumpSysprops() throws Throwable {
+def sysprops = System.properties
+TreeSet sorted = new TreeSet();
+sysprops.keys().each { String it -> sorted.add(it)}
+sorted.each { String key ->
+ log.info("$key=\"${sysprops[key]}\"")
+}
+
+ }
+}
[11/19] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17
Branch: refs/heads/develop
Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4
Parents: d6e73cd
Author: Steve Loughran
Authored: Wed Sep 10 12:13:22 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:13:22 2014 +0100
--
.../main/java/org/apache/slider/client/SliderClient.java | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8d0978c..606a53c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -201,11 +201,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
addService(yarnClient);
super.serviceInit(conf);
-
-//here the superclass is inited; getConfig returns a non-null value
-sliderFileSystem = new SliderFileSystem(getConfig());
-YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+
}
/**
@@ -298,6 +294,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
if (coreAction.getHadoopServicesRequired()) {
// validate the client
SliderUtils.validateSliderClientEnvironment(null);
+
+ //here the superclass is inited; getConfig returns a non-null value
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ YARNRegistryClient =
+ new YARNRegistryClient(yarnClient, getUsername(), getConfig());
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
[18/19] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/feature/SLIDER-86-windows Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[05/19] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/develop Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[10/19] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8
Branch: refs/heads/develop
Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea
Parents: 1efc358
Author: Steve Loughran
Authored: Wed Sep 10 12:12:42 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:42 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 107
1 file changed, 74 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index cbc0faf..4b078e7 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -18,6 +18,8 @@
import sys
import os
import subprocess
+import time
+from threading import Thread
CONF = "conf"
@@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m -Djava.confdir=%s"
+ON_POSIX = 'posix' in sys.builtin_module_names
+
+finished = False
+DEBUG = True
+
"""
Launches slider
+Nonblocking IO on windows is "tricky" ... see
+http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+to explain the code here
+
"""
@@ -66,6 +77,20 @@ def dirMustExist(dirname):
raise Exception("Directory does not exist: %s " % dirname)
return dirname
+
+def debug(text):
+ if DEBUG: print '[DEBUG] ' + text
+
+
+def error(text):
+ print '[ERROR] ' + text
+ sys.stdout.flush()
+
+def info(text):
+ print text
+ sys.stdout.flush()
+
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -86,55 +111,71 @@ def read(pipe, line):
return line, False
+def print_output(name, src):
+ """
+ Relay the output stream to stdout line by line
+ :param name:
+ :param src: source stream
+ :return:
+ """
+
+ debug ("starting printer for %s" % name )
+ line = ""
+ while not finished:
+(line, done) = read(src, line)
+if done:
+ info(name +': ' + line)
+ line = ""
+ src.close()
+
+
def runProcess(commandline):
"""
Run a process
:param commandline: command line
:return:the return code
"""
- print "ready to exec : %s" % commandline
+ debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
+ shell=False,
+ bufsize=1,
+ close_fds=ON_POSIX)
+
+ t = Thread(target=print_output, args=("stdout", exe.stdout))
+ t.daemon = True
+ t.start()
+ t2 = Thread(target=print_output, args=("stderr", exe.stderr,))
+ t2.daemon = True
+ t2.start()
+
+ debug("Waiting for completion")
while exe.poll() is None:
# process is running; grab output and echo every line
-outline, done = read(stdout, outline)
-if done:
- print outline
- outline = ""
-errline, done = read(stderr, errline)
-if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
+time.sleep(1)
+ debug("completed with exit code : %d" % exe.returncode)
return exe.returncode
-def which(program):
-def is_exe(fpath):
-return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-fpath, fname = os.path.split(program)
-if fpath:
-if is_exe(program):
-return program
-else:
-for path in os.environ["PATH"].split(os.pathsep):
-path = path.strip('"')
-exe_file = os.path.join(path, program)
-if is_exe(exe_file):
-return exe_file
+def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+def which(program):
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+if is_exe(program):
+ return program
+ else:
+for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+return exe_file
-retu
[14/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34
Branch: refs/heads/develop
Commit: f2270e34b610f7e7754bc51226f66d30ca55c581
Parents: f7906fa
Author: Steve Loughran
Authored: Wed Sep 10 13:01:14 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:01:14 2014 +0100
--
.../funtest/framework/CommandTestBase.groovy| 2 +-
.../slider/funtest/framework/SliderShell.groovy | 84 ++--
2 files changed, 77 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index b715e38..216b68d 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON :
SLIDER_SCRIPT
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 3d856c7..e49aee4 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory
class SliderShell extends Shell {
private static final Logger log = LoggerFactory.getLogger(SliderShell.class);
-
+ private static final Logger LOG = log;
public static final String BASH = '/bin/bash -s'
- public static final String CMD = 'cmd /c'
+ public static final String CMD = 'cmd'
/**
* Configuration directory, shared across all instances. Not marked as
volatile,
@@ -38,7 +38,7 @@ class SliderShell extends Shell {
*/
public static File confDir;
- public static File script;
+ public static File scriptFile;
public static final List slider_classpath_extra = []
@@ -51,8 +51,8 @@ class SliderShell extends Shell {
SliderShell(Collection commands) {
super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH)
assert confDir != null;
-assert script != null;
-command = script.absolutePath + " " + commands.join(" ")
+assert scriptFile != null;
+command = scriptFile.absolutePath + " " + commands.join(" ")
}
/**
@@ -67,7 +67,9 @@ class SliderShell extends Shell {
]
if (!slider_classpath_extra.empty) {
commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
- SliderUtils.join(slider_classpath_extra, ":", false))
+ SliderUtils.join(slider_classpath_extra,
+ (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"),
+ false))
}
commandLine << command
String script = commandLine.join("\n")
@@ -77,8 +79,18 @@ class SliderShell extends Shell {
return ret;
}
+ /**
+ * Add an environment variable
+ * @param var variable
+ * @param val value (which will be stringified)
+ * @return an env variable command
+ */
String env(String var, Object val) {
-return "export " + var + "=${val.toString()};"
+if (org.apache.hadoop.util.Shell.WINDOWS) {
+ return "set " + var + "=${val.toString()};"
+} else {
+ return "export " + var + "=${val.toString()};"
+}
}
/**
@@ -152,6 +164,62 @@ class SliderShell extends Shell {
}
}
-
+ /**
+ * Execute shell script consisting of as many Strings as we have arguments,
+ * possibly under an explicit username (requires sudoers privileges).
+ * NOTE: individual strings are conc
[12/19] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1
Branch: refs/heads/develop
Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f
Parents: a6526a1
Author: Steve Loughran
Authored: Wed Sep 10 12:25:32 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:32 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 37 ++--
1 file changed, 27 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 606a53c..ef0e63c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -196,9 +196,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-//create the YARN client
-yarnClient = new SliderYarnClientImpl();
-addService(yarnClient);
+
super.serviceInit(conf);
@@ -292,13 +290,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
- // validate the client
- SliderUtils.validateSliderClientEnvironment(null);
-
- //here the superclass is inited; getConfig returns a non-null value
- sliderFileSystem = new SliderFileSystem(getConfig());
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ initHadoopBinding();
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
@@ -353,6 +345,31 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+
+ /**
+ * Perform everything needed to init the hadoop binding.
+ * This assumes that the service is already in inited or started state
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected void initHadoopBinding() throws IOException, SliderException {
+// validate the client
+SliderUtils.validateSliderClientEnvironment(null);
+//create the YARN client
+yarnClient = new SliderYarnClientImpl();
+yarnClient.init(getConfig());
+if (getServiceState() == STATE.STARTED) {
+ yarnClient.start();
+}
+addService(yarnClient);
+// create the filesystem
+sliderFileSystem = new SliderFileSystem(getConfig());
+
+// and the registry
+YARNRegistryClient =
+new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ }
+
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
[06/19] git commit: SLIDER-413 fixed renewal logic to read token expiry time
SLIDER-413 fixed renewal logic to read token expiry time
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce0c0527
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce0c0527
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce0c0527
Branch: refs/heads/feature/SLIDER-86-windows
Commit: ce0c05278aa4d7718e3005d756cabff747f3fc5a
Parents: 809dc78
Author: Jon Maron
Authored: Tue Sep 9 18:14:09 2014 -0400
Committer: Jon Maron
Committed: Tue Sep 9 18:14:09 2014 -0400
--
.../security/FsDelegationTokenManager.java | 33 +---
.../TestFsDelegationTokenManager.groovy | 3 +-
2 files changed, 30 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce0c0527/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index 15d858c..cce2de0 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.Time;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.SliderAppMaster;
@@ -34,6 +36,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.text.DateFormat;
+import java.util.Date;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -129,6 +133,7 @@ public class FsDelegationTokenManager {
class RenewAction extends AsyncAction {
Configuration configuration;
Token token;
+private long tokenExpiryTime;
private final FileSystem fs;
RenewAction(String name,
@@ -145,14 +150,23 @@ public class FsDelegationTokenManager {
public Token run() throws Exception {
log.info("Obtaining HDFS delgation token with user {}",
remoteUser.getShortUserName());
-return fs.getDelegationToken(
+Token token = fs.getDelegationToken(
remoteUser.getShortUserName());
+tokenExpiryTime = getTokenExpiryTime(token);
+log.info("Initial delegation token obtained with expiry time of
{}", getPrintableExirationTime(tokenExpiryTime));
+return token;
}
});
}
log.info("Initial request returned delegation token {}", token);
}
+private long getTokenExpiryTime(Token token) throws IOException {
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier)token.decodeIdentifier();
+ return id.getMaxDate();
+}
+
protected FileSystem getFileSystem()
throws IOException, InterruptedException {
// return non-cache FS reference
@@ -177,11 +191,12 @@ public class FsDelegationTokenManager {
@Override
public Long run() throws Exception {
long expires = token.renew(fs.getConf());
-log.info("HDFS delegation token renewed");
+log.info("HDFS delegation token renewed. Renewal cycle ends
at {}",
+ getPrintableExirationTime(expires));
return expires;
}
});
-long calculatedInterval = expires - Time.now();
+long calculatedInterval = tokenExpiryTime - Time.now();
if ( calculatedInterval < renewInterval ) {
// time to get a new token since the token will expire before
// next renewal interval. Could modify this to be closer to
expiry
@@ -201,6 +216,11 @@ public class FsDelegationTokenManager {
}
}
+private String getPrintableExirationTime(long expires) {
+ Date d = new Date(expires);
+ return DateFormat.getDateTimeInstance().format(d);
+}
+
private void getNewToken()
throws InterruptedException, IOException {
try {
@@ -215,10 +235,13 @@ public class FsDelegationTokenManager {
throw new IOException("addDelegationTokens returned no tokens");
[01/19] git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
Repository: incubator-slider
Updated Branches:
refs/heads/develop 3c1d9dc2f -> 4a0b783ba
refs/heads/feature/SLIDER-86-windows 16d625e48 -> 4a0b783ba
SLIDER-414 RAT is creating a dir that cannot be deleted
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1
Branch: refs/heads/develop
Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef
Parents: 65d17d8
Author: Steve Loughran
Authored: Tue Sep 9 18:26:52 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 18:26:52 2014 +0100
--
.../command-logger/application-pkg/pom.xml | 38 ---
app-packages/command-logger/slider-pkg/pom.xml | 38 ---
pom.xml | 67 ++--
slider-agent/pom.xml| 62 ++-
slider-core/pom.xml | 102 +--
slider-funtest/pom.xml | 48 +
.../hbase/slider-hbase-provider/pom.xml | 49 +
src/test/clusters/script.md | 2 +-
8 files changed, 208 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml
--
diff --git a/app-packages/command-logger/application-pkg/pom.xml
b/app-packages/command-logger/application-pkg/pom.xml
index 77f0fc9..051ac84 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -54,20 +54,8 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
+
org.apache.maven.wagon
@@ -75,4 +63,28 @@
+
+
+
+ rat
+
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml
--
diff --git a/app-packages/command-logger/slider-pkg/pom.xml
b/app-packages/command-logger/slider-pkg/pom.xml
index b9bbf06..4550ae1 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -93,20 +93,6 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
@@ -118,5 +104,29 @@
tar
+
+
+
+ apache-release
+
+
+
+
+rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 167b9ef..df8599f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,36 +349,7 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
-**/*.json
-**/*.tar
-**/build.properties
-**/regionservers
-**/slaves
-**/httpfs-signature.secret
-**/dfs.exclude
-**/*.iml
-**/rat.txt
-DISCLAIMER
-app-packages/hbase/target/**
-target/*
-
-
-
-
+
@@ -1263,6 +1234,42 @@
+ rat
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
[02/19] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0
Branch: refs/heads/develop
Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:23:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:23:54 2014 +0100
--
slider-funtest/pom.xml | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml
--
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 124251f..2b55359 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -109,6 +109,12 @@
false
${build.redirect.test.output.to.file}
+
+
+ slider.bin.dir
+
../slider-assembly/target/slider-${project.version}-all/slider-${project.version}
+
+
true
true
[03/19] git commit: SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted)
SLIDER-412 Change hbase-app-package profile for HBase app package to use local
HBase tar ball (Sumit and Ted)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/809dc786
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/809dc786
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/809dc786
Branch: refs/heads/feature/SLIDER-86-windows
Commit: 809dc7860933a383b294b523011162c0872e167b
Parents: 65d17d8
Author: tedyu
Authored: Tue Sep 9 11:27:36 2014 -0700
Committer: tedyu
Committed: Tue Sep 9 11:27:36 2014 -0700
--
app-packages/hbase/README.txt | 43 ++--
app-packages/hbase/appConfig.json | 4 +-
app-packages/hbase/metainfo.xml | 4 +-
app-packages/hbase/pom.xml| 317 +++--
app-packages/hbase/src/assembly/hbase.xml | 16 +-
app-packages/storm/README.txt | 20 +-
6 files changed, 227 insertions(+), 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/README.txt
--
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
index 1d5c4bb..d0e3067 100644
--- a/app-packages/hbase/README.txt
+++ b/app-packages/hbase/README.txt
@@ -22,10 +22,26 @@ work well as the default configuration for Slider apps. So
it is advisable that
when you create an application package for Slider, include sample/default
resources.json and appConfig.json for a minimal Yarn cluster.
-OPTION-I: Use mvn command
-OPTION-II: Manual
+OPTION-I: Use a downloaded tarball
+OPTION-II: Use the tarball from the local repository
-** OPTION - I (use mvn command) **
+** OPTION - I **
+To create the app package you will need the HBase tarball and invoke mvn
command
+with appropriate parameters.
+
+Command:
+mvn clean package -Phbase-app-package -Dpkg.version=
+ -Dpkg.name= -Dpkg.src=
+
+Example:
+mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+ -Dpkg.name=hbase-0.98.5-hadoop2-bin.tar.gz
+ -Dpkg.src=/Users/user1/Downloads/0.98.5-hadoop2
+
+App package can be found in
+ app-packages/hbase/target/slider-hbase-app-package-${pkg.version}.zip
+
+** OPTION - II **
You need the HBase version available on local maven repo to create the Slider
App Package for HBase.
Download the tarball for HBase:
@@ -47,29 +63,12 @@ After HBase tarball is published locally in maven
repository, you can use the fo
App package can be found in
app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
-Verify the content using
- zip -Tv apache-slider-hbase-*.zip
-
If an HBase version older than 0.98.3 is desired, it must be installed in the
local maven repo.
A less descriptive file name can be specified with
-Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip.
-** OPTION - II (manual) **
-The Slider App Package for HBase can also be created manually.
-
-Download the tarball for HBase:
- e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz
-
-Copy the hbase tarball to package/files
- cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files
-
-Edit appConfig.json/metainfo.xml
- Replace 4 occurrences of "${hbase.version}" with the hbase version values
such as "0.98.3-hadoop2"
- Replace 1 occurrence of "${app.package.name}" with the desired app package
name, e.g. "hbase-v098"
-
-Create a zip package at the root of the package (/app-packages/hbase/)
- zip -r hbase-v098.zip .
+** Verifying the content **
Verify the content using
- zip -Tv hbase-v098.zip
+ zip -Tv apache-slider-hbase-*.zip
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 07d1b4e..3743602 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
-"application.def": "${app.package.name}.zip",
+"application.def": "slider-hbase-app-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"system_configs": "core-site",
"site.global.app_user": "yarn",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
http://git-wip-us.apache.org/repos/asf/inc
[17/19] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4
Branch: refs/heads/develop
Commit: 16d625e4800d2a64b22f62810a832120c09869f3
Parents: 682ae8a
Author: Steve Loughran
Authored: Wed Sep 10 13:45:35 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:45:35 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 921adbf..70f119e 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -145,6 +145,7 @@ class FileUploader {
public void attemptToCreateHomeDir(String username, Path home) {
def privilegedFS = getFileSystemAsUserName(username)
+log.info "Creating home dir $home as user ${user.userName} group
${user.primaryGroupName}"
privilegedFS.mkdirs(home, new FsPermission((short) 00755))
privilegedFS.setOwner(home, user.userName, user.primaryGroupName)
}
[36/36] git commit: Merge branch 'feature/SLIDER-86-windows' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'feature/SLIDER-86-windows' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/client/SliderClient.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f6bbb8b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f6bbb8b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f6bbb8b8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f6bbb8b8574c111acda2cc51918b00ed85440d60 Parents: 5784c4a 4a0b783 Author: Steve Loughran Authored: Wed Sep 10 16:43:51 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 16:43:51 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 .../command-logger/application-pkg/pom.xml | 38 ++- app-packages/command-logger/slider-pkg/pom.xml | 38 ++- app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 9 +- app-packages/storm/appConfig.json | 8 +- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/service.py | 47 ++- app-packages/storm/package/scripts/storm.py | 9 +- .../storm/package/scripts/yaml_config.py| 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 +- app-packages/storm/resources.json | 2 +- pom.xml | 78 ++--- slider-agent/pom.xml| 62 ++-- slider-assembly/pom.xml | 23 +- slider-assembly/src/main/scripts/slider | 2 + slider-assembly/src/main/scripts/slider.py | 119 --- slider-core/pom.xml | 102 ++ .../org/apache/slider/client/SliderClient.java | 92 +- .../org/apache/slider/common/SliderKeys.java| 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 + .../conf/TemplateInputPropertiesValidator.java | 2 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../providers/agent/AgentProviderService.java | 15 +- .../security/FsDelegationTokenManager.java | 38 ++- .../actions/TestActionInstallPackage.groovy | 157 + .../TestFsDelegationTokenManager.groovy | 3 +- .../agent/TestAgentProviderService.java | 10 +- slider-funtest/pom.xml | 48 +-- .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 7 +- .../funtest/framework/FileUploader.groovy | 1 + .../funtest/framework/FuntestProperties.groovy | 1 + .../slider/funtest/framework/SliderShell.groovy | 85 - .../slider/funtest/basic/SyspropsIT.groovy | 50 +++ .../hbase/slider-hbase-provider/pom.xml | 49 +-- src/test/clusters/script.md | 2 +- 58 files changed, 1394 insertions(+), 533 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java index
[15/36] git commit: SLIDER-409 AgentProviderService#publishLogFolderPaths() makes inefficient use of keySet iterator
SLIDER-409 AgentProviderService#publishLogFolderPaths() makes inefficient use
of keySet iterator
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1d083aee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1d083aee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1d083aee
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1d083aee68cb989e7e61913398a46729caed3a05
Parents: bff5ed6
Author: tedyu
Authored: Mon Sep 8 17:14:21 2014 -0700
Committer: tedyu
Committed: Mon Sep 8 17:14:21 2014 -0700
--
.../slider/providers/agent/AgentProviderService.java | 10 ++
.../services/security/FsDelegationTokenManager.java | 1 +
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1d083aee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 063e61c..9d1df61 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -840,8 +840,9 @@ public class AgentProviderService extends
AbstractProviderService implements
*/
protected void publishLogFolderPaths(
Map folders, String containerId, String roleName, String
hostFqdn) {
-for (String key : folders.keySet()) {
- workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, key,
containerId), folders.get(key));
+for (Map.Entry entry: folders.entrySet()) {
+ workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn,
entry.getKey(), containerId),
+entry.getValue());
}
publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
@@ -1392,8 +1393,9 @@ public class AgentProviderService extends
AbstractProviderService implements
for (String configType : configurations.keySet()) {
Map configBucket = configurations.get(configType);
- for (String configName : configBucket.keySet()) {
-String configValue = configBucket.get(configName);
+ for (Map.Entry entry: configBucket.entrySet()) {
+String configName = entry.getKey();
+String configValue = entry.getValue();
for (String lookUpKey : allConfigs.keySet()) {
if (configValue != null && configValue.contains(lookUpKey)) {
configValue = configValue.replace(lookUpKey,
allConfigs.get(lookUpKey));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1d083aee/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index 8f0de3d..15d858c 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -215,6 +215,7 @@ public class FsDelegationTokenManager {
throw new IOException("addDelegationTokens returned no tokens");
}
token = findMatchingToken(service, tokens);
+if (token == null) throw new IOException("Can't get new delegation
token ");
currentUser.addToken(token.getService(), token);
log.info("Expired HDFS delegation token replaced and added as
credential to current user");
[13/36] git commit: Merge branch 'develop' into feature/slider_install_package
Merge branch 'develop' into feature/slider_install_package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9af04d6d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9af04d6d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9af04d6d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 9af04d6d480080c6b917f2c55bc5569fab2ffab0 Parents: b4c88a4 bff5ed6 Author: Sumit Mohanty Authored: Mon Sep 8 14:37:01 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 14:37:01 2014 -0700 -- app-packages/storm/README.txt| 19 +++ app-packages/storm/appConfig.json| 4 ++-- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py | 6 +++--- app-packages/storm/package/scripts/service.py| 6 +++--- app-packages/storm/package/scripts/storm.py | 9 ++--- .../storm/package/scripts/yaml_config.py | 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 --- app-packages/storm/resources.json| 2 +- pom.xml | 11 ++- 10 files changed, 42 insertions(+), 38 deletions(-) --
[05/36] git commit: SLIDER-405. Minor changes to the rpm and scripts to facilitate easy deployment
SLIDER-405. Minor changes to the rpm and scripts to facilitate easy deployment
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8da15036
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8da15036
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8da15036
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8da150363571f7ab1a0b018593bc5907d501748d
Parents: 486ef84
Author: Sumit Mohanty
Authored: Sat Sep 6 08:32:07 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Sep 6 08:32:07 2014 -0700
--
slider-assembly/pom.xml| 23 +--
slider-assembly/src/main/scripts/slider| 2 ++
slider-assembly/src/main/scripts/slider.py | 10 +++---
3 files changed, 30 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8da15036/slider-assembly/pom.xml
--
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index dbbd722..c2d0ecb 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -33,7 +33,7 @@
${rpm.basedir}/bin
${rpm.basedir}/lib
${rpm.basedir}/agent
-mapred
+root
hadoop
src/conf-hdp
${project.build.directory}/lib
@@ -287,10 +287,10 @@
${rpm.agentdir}
- true
0755
${rpm.username}
${rpm.groupname}
+ false
${project.build.directory}/agent
@@ -308,6 +308,7 @@
0755
${rpm.username}
${rpm.groupname}
+ false
${src.agent.ini.dir}
@@ -318,6 +319,24 @@
+
+
+
+ ${rpm.agentdir}
+ 0755
+ ${rpm.username}
+ ${rpm.groupname}
+ true
+
+
+
+ ${rpm.agentdir}/conf
+ 0755
+ ${rpm.username}
+ ${rpm.groupname}
+ true
+
+
[06/36] git commit: more security changes
more security changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2f2d796
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2f2d796
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2f2d796
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b2f2d796b92455bba0dae9bcc1d8dc8fd6a6e7db
Parents: f4871c6
Author: Jon Maron
Authored: Sun Sep 7 13:53:16 2014 -0400
Committer: Jon Maron
Committed: Sun Sep 7 13:53:16 2014 -0400
--
app-packages/storm/package/scripts/params.py| 7 ---
app-packages/storm/package/scripts/service.py | 2 +-
app-packages/storm/package/scripts/yaml_config.py | 2 +-
app-packages/storm/package/templates/storm_jaas.conf.j2 | 11 ---
4 files changed, 10 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/params.py
--
diff --git a/app-packages/storm/package/scripts/params.py
b/app-packages/storm/package/scripts/params.py
index a8e36b9..e0bf9e4 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -48,15 +48,16 @@ if ganglia_installed:
ganglia_port = config['configurations']['global']['ganglia_server_port']
security_enabled = config['configurations']['global']['security_enabled']
-hostname_lowercase = config['hostname'].lower()
if security_enabled:
+ _hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
_storm_client_principal_name =
config['configurations']['storm-env']['storm_client_principal_name']
_storm_server_principal_name =
config['configurations']['storm-env']['storm_server_principal_name']
- storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST',
hostname_lowercase)
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST',
_hostname_lowercase)
storm_client_keytab_path =
config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',
hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',
_hostname_lowercase)
+ storm_jaas_stormclient_servicename =
storm_jaas_server_principal.split("/")[0]
storm_server_keytab_path =
config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin",
"/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/service.py
--
diff --git a/app-packages/storm/package/scripts/service.py
b/app-packages/storm/package/scripts/service.py
index 50ac44c..0ec6413 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -50,7 +50,7 @@ def service(
if name == "rest_api":
cmd = format("{rest_process_cmd} {rest_api_conf_file} >
{log_dir}/restapi.log")
else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin
STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} >
{log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH
STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} >
{log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/yaml_config.py
--
diff --git a/app-packages/storm/package/scripts/yaml_config.py
b/app-packages/storm/package/scripts/yaml_config.py
index 0a1ffa9..5f763cc 100644
--- a/app-packages/storm/package/scripts/yaml_config.py
+++ b/app-packages/storm/package/scripts/yaml_config.py
@@ -38,7 +38,7 @@ def escape_yaml_propetry(value):
# if is map {'a':'b'}
if re.match('^\w*\{.+\}\w*$', value):
- unquouted = True
+unquouted = True
try:
int(value)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/templates/storm_jaas.conf.j2
--
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2
b/app-packages/storm/package/templates/storm_jaas.conf.j2
index e68cd57..a1ba6ea 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -20,8 +2
[28/36] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a6526a17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a6526a17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a6526a17
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a6526a17fe9223bf2d2c347b939bdffdd5c8ffc4
Parents: d6e73cd
Author: Steve Loughran
Authored: Wed Sep 10 12:13:22 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:13:22 2014 +0100
--
.../main/java/org/apache/slider/client/SliderClient.java | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a6526a17/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8d0978c..606a53c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -201,11 +201,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
addService(yarnClient);
super.serviceInit(conf);
-
-//here the superclass is inited; getConfig returns a non-null value
-sliderFileSystem = new SliderFileSystem(getConfig());
-YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+
}
/**
@@ -298,6 +294,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
if (coreAction.getHadoopServicesRequired()) {
// validate the client
SliderUtils.validateSliderClientEnvironment(null);
+
+ //here the superclass is inited; getConfig returns a non-null value
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ YARNRegistryClient =
+ new YARNRegistryClient(yarnClient, getUsername(), getConfig());
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
[30/36] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7906fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7906fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7906fa3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f7906fa3a6a5fe04113832b747cbe378e5949219
Parents: d51ce5a
Author: Steve Loughran
Authored: Wed Sep 10 12:25:41 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:41 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7906fa3/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index 4b078e7..735e09b 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -135,6 +135,7 @@ def runProcess(commandline):
:param commandline: command line
:return:the return code
"""
+ global finished
debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
@@ -156,6 +157,7 @@ def runProcess(commandline):
# process is running; grab output and echo every line
time.sleep(1)
debug("completed with exit code : %d" % exe.returncode)
+ finished = True
return exe.returncode
[18/36] git commit: SLIDER-414 RAT is creating a dir that cannot be deleted
SLIDER-414 RAT is creating a dir that cannot be deleted
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/be57d7a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/be57d7a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/be57d7a1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: be57d7a11c8cc26273dc9ada64e934c5055874ef
Parents: 65d17d8
Author: Steve Loughran
Authored: Tue Sep 9 18:26:52 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 18:26:52 2014 +0100
--
.../command-logger/application-pkg/pom.xml | 38 ---
app-packages/command-logger/slider-pkg/pom.xml | 38 ---
pom.xml | 67 ++--
slider-agent/pom.xml| 62 ++-
slider-core/pom.xml | 102 +--
slider-funtest/pom.xml | 48 +
.../hbase/slider-hbase-provider/pom.xml | 49 +
src/test/clusters/script.md | 2 +-
8 files changed, 208 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/application-pkg/pom.xml
--
diff --git a/app-packages/command-logger/application-pkg/pom.xml
b/app-packages/command-logger/application-pkg/pom.xml
index 77f0fc9..051ac84 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -54,20 +54,8 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
+
org.apache.maven.wagon
@@ -75,4 +63,28 @@
+
+
+
+ rat
+
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/app-packages/command-logger/slider-pkg/pom.xml
--
diff --git a/app-packages/command-logger/slider-pkg/pom.xml
b/app-packages/command-logger/slider-pkg/pom.xml
index b9bbf06..4550ae1 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -93,20 +93,6 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
@@ -118,5 +104,29 @@
tar
+
+
+
+ apache-release
+
+
+
+
+rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/be57d7a1/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 167b9ef..df8599f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,36 +349,7 @@
-
-org.apache.rat
-apache-rat-plugin
-${apache-rat-plugin.version}
-
-
-check-licenses
-
- check
-
-
-
-
-
-**/*.json
-**/*.tar
-**/build.properties
-**/regionservers
-**/slaves
-**/httpfs-signature.secret
-**/dfs.exclude
-**/*.iml
-**/rat.txt
-DISCLAIMER
-app-packages/hbase/target/**
-target/*
-
-
-
-
+
@@ -1263,6 +1234,42 @@
+ rat
+
+
+
+org.apache.rat
+apache-rat-plugin
+${apache-rat-plugin.version}
+
+
+check-licenses
+
+ check
+
+
+
+
+
+**/*.json
+**/*.tar
+**/build.properties
+
[31/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2270e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2270e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2270e34
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f2270e34b610f7e7754bc51226f66d30ca55c581
Parents: f7906fa
Author: Steve Loughran
Authored: Wed Sep 10 13:01:14 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:01:14 2014 +0100
--
.../funtest/framework/CommandTestBase.groovy| 2 +-
.../slider/funtest/framework/SliderShell.groovy | 84 ++--
2 files changed, 77 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index b715e38..216b68d 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -106,7 +106,7 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+SliderShell.scriptFile = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON :
SLIDER_SCRIPT
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2270e34/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 3d856c7..e49aee4 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -27,10 +27,10 @@ import org.slf4j.LoggerFactory
class SliderShell extends Shell {
private static final Logger log = LoggerFactory.getLogger(SliderShell.class);
-
+ private static final Logger LOG = log;
public static final String BASH = '/bin/bash -s'
- public static final String CMD = 'cmd /c'
+ public static final String CMD = 'cmd'
/**
* Configuration directory, shared across all instances. Not marked as
volatile,
@@ -38,7 +38,7 @@ class SliderShell extends Shell {
*/
public static File confDir;
- public static File script;
+ public static File scriptFile;
public static final List slider_classpath_extra = []
@@ -51,8 +51,8 @@ class SliderShell extends Shell {
SliderShell(Collection commands) {
super(org.apache.hadoop.util.Shell.WINDOWS? CMD : BASH)
assert confDir != null;
-assert script != null;
-command = script.absolutePath + " " + commands.join(" ")
+assert scriptFile != null;
+command = scriptFile.absolutePath + " " + commands.join(" ")
}
/**
@@ -67,7 +67,9 @@ class SliderShell extends Shell {
]
if (!slider_classpath_extra.empty) {
commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
- SliderUtils.join(slider_classpath_extra, ":", false))
+ SliderUtils.join(slider_classpath_extra,
+ (org.apache.hadoop.util.Shell.WINDOWS? ";" : ":"),
+ false))
}
commandLine << command
String script = commandLine.join("\n")
@@ -77,8 +79,18 @@ class SliderShell extends Shell {
return ret;
}
+ /**
+ * Add an environment variable
+ * @param var variable
+ * @param val value (which will be stringified)
+ * @return an env variable command
+ */
String env(String var, Object val) {
-return "export " + var + "=${val.toString()};"
+if (org.apache.hadoop.util.Shell.WINDOWS) {
+ return "set " + var + "=${val.toString()};"
+} else {
+ return "export " + var + "=${val.toString()};"
+}
}
/**
@@ -152,6 +164,62 @@ class SliderShell extends Shell {
}
}
-
+ /**
+ * Execute shell script consisting of as many Strings as we have arguments,
+ * possibly under an explicit username (requires sudoers privilege
[02/36] git commit: Temporary Storm package with Security
Temporary Storm package with Security Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0878e528 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0878e528 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0878e528 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 0878e528c78147de0aab4b251ca43afe5943e57b Parents: eb6620e Author: Sumit Mohanty Authored: Thu Aug 28 18:30:42 2014 -0700 Committer: Sumit Mohanty Committed: Thu Aug 28 18:30:42 2014 -0700 -- app-packages/storm/README.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0878e528/app-packages/storm/README.txt -- diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt index 77474f1..2bae3d2 100644 --- a/app-packages/storm/README.txt +++ b/app-packages/storm/README.txt @@ -22,8 +22,8 @@ Various configurations provided in this sample are customized for apache-storm-0 So if you use a different version you may need to edit a few config values. Replace the placeholder tarball for Storm. - cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/ - rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE + cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/ + rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE Create a zip package at the root of the package (/app-packages/storm-v0_91/) zip -r Apache_Storm_v_0_9_3.zip .
[10/36] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd16210 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd16210 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd16210 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 3dd16210520ac6cb25fd3c1eb67e3d3b7640ca0b Parents: f66c0cb Author: tedyu Authored: Mon Sep 8 09:18:17 2014 -0700 Committer: tedyu Committed: Mon Sep 8 09:18:17 2014 -0700 -- pom.xml | 7 --- 1 file changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd16210/pom.xml -- diff --git a/pom.xml b/pom.xml index d9b3813..c8607ff 100644 --- a/pom.xml +++ b/pom.xml @@ -1311,13 +1311,6 @@ - hadoop-2.4.1 - -2.4.1 - - - - branch-2 2.6.0-SNAPSHOT
[19/36] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2cdb1ba0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2cdb1ba0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2cdb1ba0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2cdb1ba0009bffa41fcb24860300db5cfc9db5f4
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:23:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:23:54 2014 +0100
--
slider-funtest/pom.xml | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2cdb1ba0/slider-funtest/pom.xml
--
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 124251f..2b55359 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -109,6 +109,12 @@
false
${build.redirect.test.output.to.file}
+
+
+ slider.bin.dir
+
../slider-assembly/target/slider-${project.version}-all/slider-${project.version}
+
+
true
true
[03/36] git commit: cheanges for secure deployment
cheanges for secure deployment
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f4871c6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f4871c6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f4871c6f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f4871c6fcd2f093a313933de63107bd7e5c372b3
Parents: 0878e52
Author: Jon Maron
Authored: Tue Sep 2 19:22:27 2014 -0400
Committer: Jon Maron
Committed: Tue Sep 2 19:22:27 2014 -0400
--
app-packages/storm/package/scripts/params.py | 9 -
app-packages/storm/package/scripts/service.py | 4 ++--
app-packages/storm/package/scripts/storm.py | 9 ++---
app-packages/storm/package/scripts/yaml_config.py | 10 +-
4 files changed, 21 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/params.py
--
diff --git a/app-packages/storm/package/scripts/params.py
b/app-packages/storm/package/scripts/params.py
index 93c6018..a8e36b9 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -47,17 +47,16 @@ if ganglia_installed:
ganglia_server = config['configurations']['global']['ganglia_server_host']
ganglia_port = config['configurations']['global']['ganglia_server_port']
-_authentication =
config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication ==
'kerberos')
+security_enabled = config['configurations']['global']['security_enabled']
+hostname_lowercase = config['hostname'].lower()
if security_enabled:
- _hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
_storm_client_principal_name =
config['configurations']['storm-env']['storm_client_principal_name']
_storm_server_principal_name =
config['configurations']['storm-env']['storm_server_principal_name']
- storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST',
_hostname_lowercase)
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST',
hostname_lowercase)
storm_client_keytab_path =
config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',
_hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',
hostname_lowercase)
storm_server_keytab_path =
config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin",
"/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/service.py
--
diff --git a/app-packages/storm/package/scripts/service.py
b/app-packages/storm/package/scripts/service.py
index dc3c733..50ac44c 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -54,10 +54,10 @@ def service(
if params.security_enabled:
if name == "nimbus":
-Execute(format("{kinit_path_local} -kt {storm_server_keytab_path}
{storm_user}"),
+Execute(format("{kinit_path_local} -kt {storm_server_keytab_path}
{storm_jaas_server_principal}"),
user=params.storm_user)
else:
-Execute(format("{kinit_path_local} -kt {storm_client_keytab_path}
{storm_user}"),
+Execute(format("{kinit_path_local} -kt {storm_client_keytab_path}
{storm_jaas_client_principal}"),
user=params.storm_user)
Execute(cmd,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/storm.py
--
diff --git a/app-packages/storm/package/scripts/storm.py
b/app-packages/storm/package/scripts/storm.py
index efca073..8ecb3a1 100644
--- a/app-packages/storm/package/scripts/storm.py
+++ b/app-packages/storm/package/scripts/storm.py
@@ -50,6 +50,9 @@ def storm():
)
if params.security_enabled:
-TemplateConfig( format("{conf_dir}/storm_jaas.conf"),
- owner = params.storm_user
-)
\ No newline at end of file
+File(format("{conf_dir}/storm_jaas.conf"),
+ content=Template("storm_jaas.conf.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+)
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/yaml_config.py
[07/36] git commit: jaas fix for StormServer
jaas fix for StormServer Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4b2f8aad Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4b2f8aad Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4b2f8aad Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4b2f8aad8d924cd1f2396b0057c885028210310b Parents: b2f2d79 8da1503 Author: Jon Maron Authored: Mon Sep 8 09:27:56 2014 -0400 Committer: Jon Maron Committed: Mon Sep 8 09:27:56 2014 -0400 -- README.md | 4 +- app-packages/accumulo/appConfig.json| 7 +- .../src/test/resources/appConfig_kerberos.json | 57 ++ .../test/resources/appConfig_monitor_ssl.json | 7 +- .../src/test/resources/appConfig_ssl.json | 7 +- app-packages/hbase/appConfig.json | 76 +-- app-packages/hbase/pom.xml | 2 +- app-packages/storm/README.txt | 2 +- app-packages/storm/appConfig.json | 13 +- app-packages/storm/metainfo.xml | 26 +- ...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 - app-packages/storm/package/scripts/params.py| 8 +- app-packages/storm/package/scripts/service.py | 43 +- .../storm/package/scripts/status_params.py | 4 +- .../storm/package/templates/config.yaml.j2 | 9 - app-packages/storm/pom.xml | 90 +++ app-packages/storm/resources.json | 12 +- app-packages/storm/src/assembly/storm.xml | 68 +++ pom.xml | 42 +- slider-agent/pom.xml| 4 +- .../src/main/python/agent/Controller.py | 1 + .../python/agent/CustomServiceOrchestrator.py | 10 +- slider-agent/src/main/python/agent/Register.py | 5 +- slider-agent/src/main/python/setup.py | 2 +- .../src/test/python/agent/TestRegistration.py | 5 +- slider-agent/src/test/python/python-wrap| 40 -- slider-agent/src/test/python/python-wrap.sh | 42 ++ slider-agent/src/test/python/unitTests.py | 36 +- slider-assembly/pom.xml | 23 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy| 4 +- slider-assembly/src/main/scripts/slider | 2 + slider-assembly/src/main/scripts/slider.py | 12 +- slider-core/pom.xml | 4 +- .../org/apache/slider/api/InternalKeys.java | 11 + .../org/apache/slider/api/ResourceKeys.java | 13 - .../org/apache/slider/client/SliderClient.java | 43 +- .../common/params/AbstractActionArgs.java | 11 + .../slider/common/params/ActionFreezeArgs.java | 2 +- .../slider/common/params/ActionHelpArgs.java| 14 +- .../slider/common/params/ActionVersionArgs.java | 12 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 8 +- .../apache/slider/common/tools/SliderUtils.java | 571 ++- .../slider/core/build/InstanceBuilder.java | 5 +- .../apache/slider/core/zk/ZookeeperUtils.java | 3 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 189 +++--- .../server/appmaster/RoleLaunchService.java | 8 +- .../server/appmaster/SliderAppMaster.java | 71 ++- .../server/appmaster/actions/ActionHalt.java| 6 +- .../appmaster/actions/ActionKillContainer.java | 20 +- .../appmaster/actions/ActionStartContainer.java | 8 +- .../appmaster/actions/ActionStopSlider.java | 6 +- .../server/appmaster/actions/AsyncAction.java | 47 +- .../actions/ProviderReportedContainerLoss.java | 13 +- .../actions/ProviderStartupCompleted.java | 4 - .../server/appmaster/actions/QueueAccess.java | 5 + .../server/appmaster/actions/QueueService.java | 11 +- .../actions/RegisterComponentInstance.java | 5 + .../actions/ReviewAndFlexApplicationSize.java | 43 ++ .../actions/UnregisterComponentInstance.java| 7 +- .../server/appmaster/monkey/ChaosEntry.java | 2 +- .../slider/server/appmaster/state/AppState.java | 11 +- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +- .../appmaster/web/rest/agent/Register.java | 13 + .../services/utility/EndOfServiceWaiter.java| 84 +++ .../services/workflow/ForkedProcessService.java | 21 +- .../services/workflow/LongLivedProcess.java | 57 +- .../agent/AgentMiniClusterTestBase.groovy | 9 + .../agent/actions/TestActionExists.groovy | 2 +- .../agent/actions/TestActionVersion.groovy | 5 +- .../agent/freezethaw/TestFreezeCommands.groovy | 10 +- .../TestFreezeThawMasterl
[12/36] git commit: SLIDER-181. Storm package updates (fixing some auto-build issues)
SLIDER-181. Storm package updates (fixing some auto-build issues)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bff5ed6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bff5ed6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bff5ed6d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bff5ed6d2994978a08ea709fcc83d35aa8c94566
Parents: 34954fd
Author: Sumit Mohanty
Authored: Mon Sep 8 09:49:20 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Sep 8 09:49:20 2014 -0700
--
app-packages/storm/appConfig.json | 2 +-
app-packages/storm/metainfo.xml | 2 +-
app-packages/storm/resources.json | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index 33e38ac..bb607e9 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports":
"[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
-"site.storm-site.supervisor.childopts": "-Xmx256m
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.port=0
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+"site.storm-site.supervisor.childopts": "-Xmx256m
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.port=0
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port":
"${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/metainfo.xml
--
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 2c314b4..d201f29 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
tarball
-files/apache-storm-0.9.1.2.1.1.0-237.tar.gz
+files/apache-storm-${pkg.version}.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/resources.json
--
diff --git a/app-packages/storm/resources.json
b/app-packages/storm/resources.json
index f14b2c8..edfdc1f 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -13,7 +13,7 @@
},
"STORM_UI_SERVER": {
"yarn.role.priority": "2",
- "yarn.component.instances": "0"
+ "yarn.component.instances": "1"
},
"DRPC_SERVER": {
"yarn.role.priority": "3",
[23/36] git commit: SLIDER-413 fixed renewal logic to read token expiry time
SLIDER-413 fixed renewal logic to read token expiry time
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce0c0527
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce0c0527
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce0c0527
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ce0c05278aa4d7718e3005d756cabff747f3fc5a
Parents: 809dc78
Author: Jon Maron
Authored: Tue Sep 9 18:14:09 2014 -0400
Committer: Jon Maron
Committed: Tue Sep 9 18:14:09 2014 -0400
--
.../security/FsDelegationTokenManager.java | 33 +---
.../TestFsDelegationTokenManager.groovy | 3 +-
2 files changed, 30 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce0c0527/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index 15d858c..cce2de0 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.Time;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.SliderAppMaster;
@@ -34,6 +36,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.text.DateFormat;
+import java.util.Date;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -129,6 +133,7 @@ public class FsDelegationTokenManager {
class RenewAction extends AsyncAction {
Configuration configuration;
Token token;
+private long tokenExpiryTime;
private final FileSystem fs;
RenewAction(String name,
@@ -145,14 +150,23 @@ public class FsDelegationTokenManager {
public Token run() throws Exception {
log.info("Obtaining HDFS delgation token with user {}",
remoteUser.getShortUserName());
-return fs.getDelegationToken(
+Token token = fs.getDelegationToken(
remoteUser.getShortUserName());
+tokenExpiryTime = getTokenExpiryTime(token);
+log.info("Initial delegation token obtained with expiry time of
{}", getPrintableExirationTime(tokenExpiryTime));
+return token;
}
});
}
log.info("Initial request returned delegation token {}", token);
}
+private long getTokenExpiryTime(Token token) throws IOException {
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier)token.decodeIdentifier();
+ return id.getMaxDate();
+}
+
protected FileSystem getFileSystem()
throws IOException, InterruptedException {
// return non-cache FS reference
@@ -177,11 +191,12 @@ public class FsDelegationTokenManager {
@Override
public Long run() throws Exception {
long expires = token.renew(fs.getConf());
-log.info("HDFS delegation token renewed");
+log.info("HDFS delegation token renewed. Renewal cycle ends
at {}",
+ getPrintableExirationTime(expires));
return expires;
}
});
-long calculatedInterval = expires - Time.now();
+long calculatedInterval = tokenExpiryTime - Time.now();
if ( calculatedInterval < renewInterval ) {
// time to get a new token since the token will expire before
// next renewal interval. Could modify this to be closer to
expiry
@@ -201,6 +216,11 @@ public class FsDelegationTokenManager {
}
}
+private String getPrintableExirationTime(long expires) {
+ Date d = new Date(expires);
+ return DateFormat.getDateTimeInstance().format(d);
+}
+
private void getNewToken()
throws InterruptedException, IOException {
try {
@@ -215,10 +235,13 @@ public class FsDelegationTokenManager {
throw new IOException("addDelegationTokens ret
[08/36] git commit: SLIDER-407. Add install package command to install app packages
SLIDER-407. Add install package command to install app packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b4c88a4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b4c88a4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b4c88a4b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b4c88a4b30309fb78f976c783932a133d9fe0058
Parents: 8da1503
Author: Sumit Mohanty
Authored: Mon Sep 8 06:36:04 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Sep 8 06:36:04 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 60 +--
.../org/apache/slider/common/SliderKeys.java| 2 +
.../common/params/AbstractActionArgs.java | 3 +-
.../common/params/ActionInstallPackageArgs.java | 58 +++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 12 +-
.../slider/common/params/SliderActions.java | 2 +
.../slider/common/tools/CoreFileSystem.java | 12 ++
.../actions/TestActionInstallPackage.groovy | 158 +++
9 files changed, 296 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef6448d..98074ec 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -48,6 +48,7 @@ import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionEchoArgs;
@@ -301,20 +302,22 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
-if (ACTION_BUILD.equals(action)) {
+if (ACTION_INSTALL_PACKAGE.equals(action)) {
+ exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
+} else if (ACTION_BUILD.equals(action)) {
exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
} else if (ACTION_CREATE.equals(action)) {
exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
} else if (ACTION_FREEZE.equals(action)) {
exitCode = actionFreeze(clusterName,
- serviceArgs.getActionFreezeArgs());
+serviceArgs.getActionFreezeArgs());
} else if (ACTION_THAW.equals(action)) {
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
} else if (ACTION_GETCONF.equals(action)) {
@@ -322,15 +325,12 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
} else if (ACTION_HELP.equals(action) ||
ACTION_USAGE.equals(action)) {
log.info(serviceArgs.usage());
-
} else if (ACTION_KILL_CONTAINER.equals(action)) {
exitCode = actionKillContainer(clusterName,
serviceArgs.getActionKillContainerArgs());
-
} else if (ACTION_AM_SUICIDE.equals(action)) {
exitCode = actionAmSuicide(clusterName,
serviceArgs.getActionAMSuicideArgs());
-
} else if (ACTION_LIST.equals(action)) {
exitCode = actionList(clusterName);
} else if (ACTION_REGISTRY.equals(action)) {
@@ -592,6 +592,50 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
/**
+ * Upload application package to user home directory
+ *
+ * @param installPkgInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
+ YarnExcep
[17/36] git commit: Merge branch 'feature/slider_install_package' into develop
Merge branch 'feature/slider_install_package' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/65d17d84 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/65d17d84 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/65d17d84 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 65d17d84487594939a904aeefc1b6f85464c35f0 Parents: f6d7917 804788c Author: Sumit Mohanty Authored: Mon Sep 8 22:18:03 2014 -0700 Committer: Sumit Mohanty Committed: Mon Sep 8 22:18:03 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 60 +-- .../org/apache/slider/common/SliderKeys.java| 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 +++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 ++ .../actions/TestActionInstallPackage.groovy | 157 +++ 9 files changed, 295 insertions(+), 12 deletions(-) --
[20/36] git commit: SLIDER-412 Change hbase-app-package profile for HBase app package to use local HBase tar ball (Sumit and Ted)
SLIDER-412 Change hbase-app-package profile for HBase app package to use local
HBase tar ball (Sumit and Ted)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/809dc786
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/809dc786
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/809dc786
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 809dc7860933a383b294b523011162c0872e167b
Parents: 65d17d8
Author: tedyu
Authored: Tue Sep 9 11:27:36 2014 -0700
Committer: tedyu
Committed: Tue Sep 9 11:27:36 2014 -0700
--
app-packages/hbase/README.txt | 43 ++--
app-packages/hbase/appConfig.json | 4 +-
app-packages/hbase/metainfo.xml | 4 +-
app-packages/hbase/pom.xml| 317 +++--
app-packages/hbase/src/assembly/hbase.xml | 16 +-
app-packages/storm/README.txt | 20 +-
6 files changed, 227 insertions(+), 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/README.txt
--
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
index 1d5c4bb..d0e3067 100644
--- a/app-packages/hbase/README.txt
+++ b/app-packages/hbase/README.txt
@@ -22,10 +22,26 @@ work well as the default configuration for Slider apps. So
it is advisable that
when you create an application package for Slider, include sample/default
resources.json and appConfig.json for a minimal Yarn cluster.
-OPTION-I: Use mvn command
-OPTION-II: Manual
+OPTION-I: Use a downloaded tarball
+OPTION-II: Use the tarball from the local repository
-** OPTION - I (use mvn command) **
+** OPTION - I **
+To create the app package you will need the HBase tarball and invoke mvn
command
+with appropriate parameters.
+
+Command:
+mvn clean package -Phbase-app-package -Dpkg.version=
+ -Dpkg.name= -Dpkg.src=
+
+Example:
+mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+ -Dpkg.name=hbase-0.98.5-hadoop2-bin.tar.gz
+ -Dpkg.src=/Users/user1/Downloads/0.98.5-hadoop2
+
+App package can be found in
+ app-packages/hbase/target/slider-hbase-app-package-${pkg.version}.zip
+
+** OPTION - II **
You need the HBase version available on local maven repo to create the Slider
App Package for HBase.
Download the tarball for HBase:
@@ -47,29 +63,12 @@ After HBase tarball is published locally in maven
repository, you can use the fo
App package can be found in
app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
-Verify the content using
- zip -Tv apache-slider-hbase-*.zip
-
If an HBase version older than 0.98.3 is desired, it must be installed in the
local maven repo.
A less descriptive file name can be specified with
-Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip.
-** OPTION - II (manual) **
-The Slider App Package for HBase can also be created manually.
-
-Download the tarball for HBase:
- e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz
-
-Copy the hbase tarball to package/files
- cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files
-
-Edit appConfig.json/metainfo.xml
- Replace 4 occurrences of "${hbase.version}" with the hbase version values
such as "0.98.3-hadoop2"
- Replace 1 occurrence of "${app.package.name}" with the desired app package
name, e.g. "hbase-v098"
-
-Create a zip package at the root of the package (/app-packages/hbase/)
- zip -r hbase-v098.zip .
+** Verifying the content **
Verify the content using
- zip -Tv hbase-v098.zip
+ zip -Tv apache-slider-hbase-*.zip
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/809dc786/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 07d1b4e..3743602 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
-"application.def": "${app.package.name}.zip",
+"application.def": "slider-hbase-app-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"system_configs": "core-site",
"site.global.app_user": "yarn",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
http://git-wip-us
[32/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/49ae1731
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/49ae1731
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/49ae1731
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 49ae17310b163e0bc0f90a0bb925f84819f36d2b
Parents: f2270e3
Author: Steve Loughran
Authored: Wed Sep 10 13:05:25 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:05:25 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/SliderShell.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/49ae1731/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index e49aee4..6effb46 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -87,7 +87,7 @@ class SliderShell extends Shell {
*/
String env(String var, Object val) {
if (org.apache.hadoop.util.Shell.WINDOWS) {
- return "set " + var + "=${val.toString()};"
+ return "set " + var + "=${val.toString()}"
} else {
return "export " + var + "=${val.toString()};"
}
[34/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/16d625e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/16d625e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/16d625e4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 16d625e4800d2a64b22f62810a832120c09869f3
Parents: 682ae8a
Author: Steve Loughran
Authored: Wed Sep 10 13:45:35 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:45:35 2014 +0100
--
.../groovy/org/apache/slider/funtest/framework/FileUploader.groovy | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/16d625e4/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 921adbf..70f119e 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -145,6 +145,7 @@ class FileUploader {
public void attemptToCreateHomeDir(String username, Path home) {
def privilegedFS = getFileSystemAsUserName(username)
+log.info "Creating home dir $home as user ${user.userName} group
${user.primaryGroupName}"
privilegedFS.mkdirs(home, new FsPermission((short) 00755))
privilegedFS.setOwner(home, user.userName, user.primaryGroupName)
}
[22/36] git commit: SLIDER-86 git reversion grief
SLIDER-86 git reversion grief Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f6f15ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f6f15ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f6f15ff Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 8f6f15ff6ee8681df574cc3e413253d591d34cf9 Parents: 41f7b85 2cdb1ba Author: Steve Loughran Authored: Tue Sep 9 19:44:45 2014 +0100 Committer: Steve Loughran Committed: Tue Sep 9 19:44:45 2014 +0100 -- --
[35/36] git commit: Merge branch 'develop' into feature/SLIDER-86-windows
Merge branch 'develop' into feature/SLIDER-86-windows Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a0b783b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a0b783b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a0b783b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4a0b783ba24dcaed93c4f6b61226359b063a0487 Parents: 16d625e 3c1d9dc Author: Steve Loughran Authored: Wed Sep 10 13:57:46 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 10 13:57:46 2014 +0100 -- app-packages/accumulo/README.txt| 17 +- app-packages/accumulo/pom.xml | 90 +- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 + .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 app-packages/hbase/README.txt | 43 ++- app-packages/hbase/appConfig.json | 4 +- app-packages/hbase/metainfo.xml | 4 +- app-packages/hbase/pom.xml | 317 +++ app-packages/hbase/src/assembly/hbase.xml | 16 +- app-packages/storm/README.txt | 20 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- 21 files changed, 561 insertions(+), 208 deletions(-) --
[26/36] git commit: SLIDER-86: slider funtests call cmd /c slider.py on windows
SLIDER-86: slider funtests call cmd /c slider.py on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1efc3583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1efc3583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1efc3583
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1efc35835bfb7615923411d420d2b61ce5d64ebc
Parents: 8f6f15f
Author: Steve Loughran
Authored: Wed Sep 10 12:12:10 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:10 2014 +0100
--
.../apache/slider/funtest/framework/CommandTestBase.groovy| 7 ++-
.../apache/slider/funtest/framework/FuntestProperties.groovy | 1 +
.../org/apache/slider/funtest/framework/SliderShell.groovy| 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 3b52912..b715e38 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
@@ -56,6 +57,9 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final File SLIDER_SCRIPT = new File(
SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
+ public static final File SLIDER_SCRIPT_PYTHON = new File(
+ SLIDER_TAR_DIRECTORY,
+ BIN_SLIDER_PYTHON).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
@@ -102,7 +106,8 @@ abstract class CommandTestBase extends SliderTestUtils {
log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
-SliderShell.script = SLIDER_SCRIPT
+SliderShell.script = Shell.WINDOWS ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @
${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 1096dfa..730639b 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,6 +57,7 @@ public interface FuntestProperties extends
SliderXMLConfKeysForTesting {
String KEY_TEST_CONF_XML = "slider.test.conf.xml"
String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
String BIN_SLIDER = "bin/slider"
+ String BIN_SLIDER_PYTHON = "bin/slider.py"
String AGENT_INI = "agent.ini"
String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1efc3583/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 804e791..3d856c7 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -30,6 +30,7 @@ class SliderShell extends Shell {
public static final String BASH = '/bin/bash -s'
+ public static final String CMD = 'cmd /c'
/**
* Configuration directory, shared across all instances. Not marked as
volatile,
@@ -48,7 +49,7 @@ class SliderShell extends Shell {
[14/36] git commit: SLIDER-407. Add install package command to install app packages
SLIDER-407. Add install package command to install app packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/804788c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/804788c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/804788c3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 804788c338e80381ae33049cdb79f4db25a378ec
Parents: 9af04d6
Author: Sumit Mohanty
Authored: Mon Sep 8 16:45:08 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Sep 8 16:45:08 2014 -0700
--
.../src/main/java/org/apache/slider/client/SliderClient.java| 2 +-
.../apache/slider/agent/actions/TestActionInstallPackage.groovy | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/804788c3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 98074ec..8d0978c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -628,7 +628,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
if (sliderFileSystem.getFileSystem().exists(fileInFs) &&
!installPkgInfo.replacePkg) {
throw new BadCommandArgumentsException("Pkg exists at " +
fileInFs.toUri().toString() +
- ". Use --replacePkg true to
overwrite.");
+ ". Use --replacePkg to
overwrite.");
}
sliderFileSystem.getFileSystem().copyFromLocalFile(false,
installPkgInfo.replacePkg, srcFile, fileInFs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/804788c3/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
index 4d89e9b..88f1298 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
@@ -103,7 +103,7 @@ class TestActionInstallPackage extends
AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " +
launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("Use --replacePkg true to overwrite")
+ assert e.message.contains("Use --replacePkg to overwrite")
}
}
@@ -147,12 +147,11 @@ class TestActionInstallPackage extends
AgentMiniClusterTestBase {
SliderActions.ACTION_INSTALL_PACKAGE,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
- Arguments.ARG_REPLACE_PKG, true
+ Arguments.ARG_REPLACE_PKG
],
)
} catch (BadCommandArgumentsException e) {
log.info(e.message)
- assert e.message.contains("Use --replacePkg true to overwrite")
}
}
}
[24/36] git commit: SLIDER-416 substitute hostname for _HOST in principals
SLIDER-416 substitute hostname for _HOST in principals
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e24060b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e24060b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e24060b9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e24060b93751f9a73592be0ca628d3a403d044a3
Parents: ce0c052
Author: Billie Rinaldi
Authored: Tue Sep 9 20:17:30 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 9 20:17:30 2014 -0700
--
.../java/org/apache/slider/core/launch/AppMasterLauncher.java | 6 +-
.../server/services/security/FsDelegationTokenManager.java | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index bd8a0a5..c9ac20c 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -20,6 +20,7 @@ package org.apache.slider.core.launch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -35,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
@@ -196,7 +198,9 @@ public class AppMasterLauncher extends AbstractLauncher {
*/
private void addSecurityTokens() throws IOException {
-String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL);
+String tokenRenewer = SecurityUtil.getServerPrincipal(
+getConf().get(YarnConfiguration.RM_PRINCIPAL),
+InetAddress.getLocalHost().getCanonicalHostName());
if (SliderUtils.isUnset(tokenRenewer)) {
throw new IOException(
"Can't get Master Kerberos principal for the RM to use as renewer: "
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e24060b9/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index cce2de0..c892b10 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -35,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
import java.security.PrivilegedExceptionAction;
import java.text.DateFormat;
import java.util.Date;
@@ -67,7 +69,9 @@ public class FsDelegationTokenManager {
// using HDFS principal...
this.remoteUser = UserGroupInformation
.loginUserFromKeytabAndReturnUGI(
-loginConfig.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
+SecurityUtil.getServerPrincipal(
+
loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
+InetAddress.getLocalHost().getCanonicalHostName()),
loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
log.info("Created remote user {}. UGI reports current user is {}",
this.remoteUser, UserGroupInformation.getCurrentUser());
[29/36] git commit: SLIDER-418 sider version command fails if the config isn't valid
SLIDER-418 sider version command fails if the config isn't valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d51ce5a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d51ce5a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d51ce5a1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d51ce5a15dfcd8d00a317e1d459b3ac1b097a64f
Parents: a6526a1
Author: Steve Loughran
Authored: Wed Sep 10 12:25:32 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:25:32 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 37 ++--
1 file changed, 27 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d51ce5a1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 606a53c..ef0e63c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -196,9 +196,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-//create the YARN client
-yarnClient = new SliderYarnClientImpl();
-addService(yarnClient);
+
super.serviceInit(conf);
@@ -292,13 +290,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
- // validate the client
- SliderUtils.validateSliderClientEnvironment(null);
-
- //here the superclass is inited; getConfig returns a non-null value
- sliderFileSystem = new SliderFileSystem(getConfig());
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ initHadoopBinding();
}
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
@@ -353,6 +345,31 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+
+ /**
+ * Perform everything needed to init the hadoop binding.
+ * This assumes that the service is already in inited or started state
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected void initHadoopBinding() throws IOException, SliderException {
+// validate the client
+SliderUtils.validateSliderClientEnvironment(null);
+//create the YARN client
+yarnClient = new SliderYarnClientImpl();
+yarnClient.init(getConfig());
+if (getServiceState() == STATE.STARTED) {
+ yarnClient.start();
+}
+addService(yarnClient);
+// create the filesystem
+sliderFileSystem = new SliderFileSystem(getConfig());
+
+// and the registry
+YARNRegistryClient =
+new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ }
+
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
[01/36] git commit: Temporary Storm package with Security
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 5784c4ab5 ->
f6bbb8b85
Temporary Storm package with Security
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb6620e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb6620e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb6620e7
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: eb6620e77a1e8c97c5b65f8d098040d03594e478
Parents: 8bfd66e
Author: Sumit Mohanty
Authored: Thu Aug 28 18:21:18 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Aug 28 18:21:18 2014 -0700
--
app-packages/storm/README.txt | 4 +--
app-packages/storm/appConfig.json | 11
app-packages/storm/configuration/storm-env.xml | 25 ++
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16
...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16
app-packages/storm/package/scripts/params.py| 11 +---
app-packages/storm/package/scripts/service.py | 8 ++
.../storm/package/templates/storm_jaas.conf.j2 | 27 +---
app-packages/storm/resources.json | 2 +-
10 files changed, 91 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/README.txt
--
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 971cf14..77474f1 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -26,10 +26,10 @@ Replace the placeholder tarball for Storm.
rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
Create a zip package at the root of the package (/app-packages/storm-v0_91/)
- zip -r storm_v091.zip .
+ zip -r Apache_Storm_v_0_9_3.zip .
Verify the content using
- unzip -l "$@" storm_v091.zip
+ unzip -l "$@" Apache_Storm_v_0_9_3.zip
While appConfig.json and resources.json are not required for the package they
work
well as the default configuration for Slider apps. So its advisable that when
you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index b8454f9..d6dee84 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,12 +3,12 @@
"metadata": {
},
"global": {
-"application.def": "storm_v091.zip",
+"application.def": "Apache_Storm_v_0_9_3.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"site.global.app_user": "yarn",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
@@ -18,13 +18,14 @@
"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
+"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
"site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
-"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
-"site.storm-site.worker.childopts": "-Xmx768m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multica
[04/36] git commit: SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python) SLIDER-402. Add support for general purpose cross referencing of na
SLIDER-181. Storm package updates (changes to accomodate storm script changing
to be a shell script from python)
SLIDER-402. Add support for general purpose cross referencing of named
properties (handle multiple replacements for single property)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/486ef84f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/486ef84f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/486ef84f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 486ef84f4cb7e6e716e70a82ac08e1c43b3c497d
Parents: f14d32c
Author: Sumit Mohanty
Authored: Fri Sep 5 20:59:04 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Sep 5 20:59:24 2014 -0700
--
app-packages/storm/appConfig.json | 6 +--
app-packages/storm/package/scripts/params.py| 2 +-
app-packages/storm/package/scripts/service.py | 43
.../providers/agent/AgentProviderService.java | 5 ++-
.../agent/TestAgentProviderService.java | 10 ++---
5 files changed, 28 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index ac9bd49..1eae599 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -23,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
-"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
-"site.storm-site.worker.childopts": "-Xmx768m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+"site.storm-site.nimbus.childopts": "-Xmx1024m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+"site.storm-site.worker.childopts": "-Xmx768m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path":
"${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports":
"[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
-"site.storm-site.supervisor.childopts": "-Xmx256m
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.port=0
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+"site.storm-site.supervisor.childopts": "-Xmx256m
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.port=0
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric
[09/36] git commit: Merge branch 'feature/update_storm_for_93' into develop
Merge branch 'feature/update_storm_for_93' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f66c0cbc Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f66c0cbc Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f66c0cbc Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f66c0cbc0f211da1ea52acf7cad8a20480a1f5ea Parents: 8da1503 4b2f8aa Author: Jon Maron Authored: Mon Sep 8 09:41:14 2014 -0400 Committer: Jon Maron Committed: Mon Sep 8 09:41:14 2014 -0400 -- app-packages/storm/README.txt| 19 +++ app-packages/storm/appConfig.json| 4 ++-- app-packages/storm/metainfo.xml | 2 +- app-packages/storm/package/scripts/params.py | 6 +++--- app-packages/storm/package/scripts/service.py| 6 +++--- app-packages/storm/package/scripts/storm.py | 9 ++--- .../storm/package/scripts/yaml_config.py | 10 +- .../storm/package/templates/storm_jaas.conf.j2 | 11 --- app-packages/storm/resources.json| 4 ++-- 9 files changed, 41 insertions(+), 30 deletions(-) --
[25/36] git commit: SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to work in secure mode
SLIDER-417, SLIDER-415 Enable local package building for accumulo, fix tests to
work in secure mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c1d9dc2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c1d9dc2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c1d9dc2
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 3c1d9dc2fcec1a897a26388865194c99fc28e2f8
Parents: e24060b
Author: Billie Rinaldi
Authored: Tue Sep 9 22:14:36 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Sep 9 22:14:36 2014 -0700
--
app-packages/accumulo/README.txt| 17 ++--
app-packages/accumulo/pom.xml | 90 ++--
app-packages/accumulo/src/assembly/accumulo.xml | 2 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++
.../accumulo/AccumuloMonitorSSLIT.groovy| 7 +-
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +-
.../accumulo/src/test/resources/appConfig.json | 55
.../src/test/resources/appConfig_kerberos.json | 8 +-
.../test/resources/appConfig_monitor_ssl.json | 4 +-
.../appConfig_monitor_ssl_kerberos.json | 57 +
.../src/test/resources/appConfig_ssl.json | 4 +-
.../test/resources/appConfig_ssl_kerberos.json | 59 +
12 files changed, 294 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/README.txt
--
diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt
index 634296a..f75814c 100644
--- a/app-packages/accumulo/README.txt
+++ b/app-packages/accumulo/README.txt
@@ -17,21 +17,28 @@
# How to create a Slider package for Accumulo?
- mvn clean package -DskipTests -Paccumulo-app-package
+ mvn clean package -DskipTests -Paccumulo-app-package-maven
+ OR
+ mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \
+-Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball
App package can be found in
-
app-packages/accumulo/target/apache-slider-accumulo-${accumulo.version}-app-package-${slider.version}.zip
+ app-packages/accumulo/target/slider-accumulo-app-package-*.zip
+
+In the first case, the version number of the app package will match the
+slider version, and in the second case it will match the pkg.version
+(intended to be the accumulo version).
Verify the content using
- zip -Tv apache-slider-accumulo-*.zip
+ zip -Tv slider-accumulo-app-package*.zip
While appConfig.json and resources.json are not required for the package they
work well as the default configuration for Slider apps. So it is advisable that
when you create an application package for Slider, include sample/default
resources.json and appConfig.json for a minimal Yarn cluster.
-The version of Accumulo used for the app package can be adjusted by adding a
-flag such as
+In the maven packaging case, the version of Accumulo used for the app package
+can be adjusted by adding a flag such as
-Daccumulo.version=1.5.1
**Note that the LICENSE.txt and NOTICE.txt that are bundled with the app
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c1d9dc2/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 548ced8..f074b01 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -30,17 +30,20 @@
package-tmp
-
${project.artifactId}-${project.version}
+${project.artifactId}-${pkg.version}
+${project.build.directory}/${work.dir}
+${project.version}
+accumulo-${accumulo.version}-bin.tar.gz
+
+yarn
+hadoop
+
+
accumulo-app-package
-
-
- !skipAccumulo
-
-
@@ -64,6 +67,79 @@
org.apache.maven.plugins
+maven-antrun-plugin
+1.7
+
+
+copy
+validate
+
+
+
+
+
+
+ run
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-failsafe-plugin
+${maven-failsafe-plugin.version}
+
+
+run-integration-tests
+
+ integration-test
+ verify
+
+
+
+
+
+true
+
[33/36] git commit: SLIDER-420 slider funtests not working on windows
SLIDER-420 slider funtests not working on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/682ae8a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/682ae8a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/682ae8a7
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 682ae8a7fa92932f32420486761efa06b5febbc1
Parents: 49ae173
Author: Steve Loughran
Authored: Wed Sep 10 13:40:17 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 13:40:17 2014 +0100
--
.../apache/slider/funtest/framework/AgentCommandTestBase.groovy| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/682ae8a7/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 50f3ac0..81d1595 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -96,7 +96,7 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
log.info "App pkg uploaded at $appPkgPath"
} catch (Exception e) {
setup_failed = true
- fail("Setup failed "+e)
+ throw e;
}
}
[11/36] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34954fd2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34954fd2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34954fd2 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 34954fd2151d6f0b42877cf9e50aacd7909136bb Parents: 3dd1621 Author: tedyu Authored: Mon Sep 8 09:20:49 2014 -0700 Committer: tedyu Committed: Mon Sep 8 09:20:49 2014 -0700 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34954fd2/pom.xml -- diff --git a/pom.xml b/pom.xml index c8607ff..167b9ef 100644 --- a/pom.xml +++ b/pom.xml @@ -1303,8 +1303,8 @@ local -2.4.1-SNAPSHOT -0.98.4-SNAPSHOT +2.6.0-SNAPSHOT +0.98.5-SNAPSHOT 1.6.0-SNAPSHOT
[27/36] git commit: SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
SLIDER-86: slider.py non-blocking streaming on all platforms; debugging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6e73cd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6e73cd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6e73cd8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d6e73cd8d06d70cba5da9b19c9b00330161cb3ea
Parents: 1efc358
Author: Steve Loughran
Authored: Wed Sep 10 12:12:42 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 12:12:42 2014 +0100
--
slider-assembly/src/main/scripts/slider.py | 107
1 file changed, 74 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6e73cd8/slider-assembly/src/main/scripts/slider.py
--
diff --git a/slider-assembly/src/main/scripts/slider.py
b/slider-assembly/src/main/scripts/slider.py
index cbc0faf..4b078e7 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -18,6 +18,8 @@
import sys
import os
import subprocess
+import time
+from threading import Thread
CONF = "conf"
@@ -31,9 +33,18 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true
-Xmx256m -Djava.confdir=%s"
+ON_POSIX = 'posix' in sys.builtin_module_names
+
+finished = False
+DEBUG = True
+
"""
Launches slider
+Nonblocking IO on windows is "tricky" ... see
+http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+to explain the code here
+
"""
@@ -66,6 +77,20 @@ def dirMustExist(dirname):
raise Exception("Directory does not exist: %s " % dirname)
return dirname
+
+def debug(text):
+ if DEBUG: print '[DEBUG] ' + text
+
+
+def error(text):
+ print '[ERROR] ' + text
+ sys.stdout.flush()
+
+def info(text):
+ print text
+ sys.stdout.flush()
+
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -86,55 +111,71 @@ def read(pipe, line):
return line, False
+def print_output(name, src):
+ """
+ Relay the output stream to stdout line by line
+ :param name:
+ :param src: source stream
+ :return:
+ """
+
+ debug ("starting printer for %s" % name )
+ line = ""
+ while not finished:
+(line, done) = read(src, line)
+if done:
+ info(name +': ' + line)
+ line = ""
+ src.close()
+
+
def runProcess(commandline):
"""
Run a process
:param commandline: command line
:return:the return code
"""
- print "ready to exec : %s" % commandline
+ debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
+ shell=False,
+ bufsize=1,
+ close_fds=ON_POSIX)
+
+ t = Thread(target=print_output, args=("stdout", exe.stdout))
+ t.daemon = True
+ t.start()
+ t2 = Thread(target=print_output, args=("stderr", exe.stderr,))
+ t2.daemon = True
+ t2.start()
+
+ debug("Waiting for completion")
while exe.poll() is None:
# process is running; grab output and echo every line
-outline, done = read(stdout, outline)
-if done:
- print outline
- outline = ""
-errline, done = read(stderr, errline)
-if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
+time.sleep(1)
+ debug("completed with exit code : %d" % exe.returncode)
return exe.returncode
-def which(program):
-def is_exe(fpath):
-return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-fpath, fname = os.path.split(program)
-if fpath:
-if is_exe(program):
-return program
-else:
-for path in os.environ["PATH"].split(os.pathsep):
-path = path.strip('"')
-exe_file = os.path.join(path, program)
-if is_exe(exe_file):
-return exe_file
+def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+def which(program):
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+if is_exe(program):
+ return program
+ else:
+for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_
[16/36] git commit: SLIDER-411 Correct error message when property whose name starts with 'yarn.' is encountered in template
SLIDER-411 Correct error message when property whose name starts with 'yarn.'
is encountered in template
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f6d7917d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f6d7917d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f6d7917d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f6d7917d250509bd751164a891ef43f770243327
Parents: 1d083ae
Author: tedyu
Authored: Mon Sep 8 20:50:10 2014 -0700
Committer: tedyu
Committed: Mon Sep 8 20:50:10 2014 -0700
--
.../apache/slider/core/conf/TemplateInputPropertiesValidator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6d7917d/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
index 5494174..aad2757 100644
---
a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
+++
b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
@@ -27,7 +27,7 @@ public class TemplateInputPropertiesValidator
void validatePropertyNamePrefix(String key) throws BadConfigException {
if (key.startsWith("yarn.")) {
throw new BadConfigException(
- "argument %s does not have 'yarn.' prefix", key);
+ "argument %s has 'yarn.' prefix - this is not allowed in templates",
key);
}
}
[21/36] git commit: SLIDER-86 failsafe not passing down sysprops
SLIDER-86 failsafe not passing down sysprops
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41f7b85b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41f7b85b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41f7b85b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 41f7b85b4520bf2ba3b6be3f572999ddd90e
Parents: be57d7a
Author: Steve Loughran
Authored: Tue Sep 9 19:43:39 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 9 19:43:39 2014 +0100
--
.../slider/funtest/basic/SyspropsIT.groovy | 50
1 file changed, 50 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41f7b85b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
--
diff --git
a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
new file mode 100644
index 000..728920e
--- /dev/null
+++
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on CommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@Slf4j
+class SyspropsIT extends SliderTestUtils{
+
+
+ @Test
+ public void testDumpSysprops() throws Throwable {
+def sysprops = System.properties
+TreeSet sorted = new TreeSet();
+sysprops.keys().each { String it -> sorted.add(it)}
+sorted.each { String key ->
+ log.info("$key=\"${sysprops[key]}\"")
+}
+
+ }
+}
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows [deleted] 4a0b783ba
Git Push Summary
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-86-windows [created] 8b32aa17a
git commit: SLIDER-381 Slider-client exists command does not report status correctly
Repository: incubator-slider
Updated Branches:
refs/heads/develop 8b32aa17a -> 7c95d5b05
SLIDER-381 Slider-client exists command does not report status correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c95d5b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c95d5b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c95d5b0
Branch: refs/heads/develop
Commit: 7c95d5b059045cc01fa9756308f50e01644abb2d
Parents: 8b32aa1
Author: Steve Loughran
Authored: Wed Sep 10 18:04:18 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 18:04:18 2014 +0100
--
.../java/org/apache/slider/client/SliderClient.java| 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c95d5b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef0e63c..7c1fd1e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1623,10 +1623,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @return exit code
*/
@VisibleForTesting
- public int actionExists(String name, boolean live) throws YarnException,
IOException {
+ public int actionExists(String name, boolean checkLive) throws
YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(name);
-log.debug("actionExists({}, {})", name, live);
+log.debug("actionExists({}, {})", name, checkLive);
//initial probe for a cluster in the filesystem
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
@@ -1636,10 +1636,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
//test for liveness if desired
-if (live) {
+if (checkLive) {
ApplicationReport instance = findInstance(name);
if (instance == null) {
-log.info("cluster {} not running", name);
+log.info("Cluster {} not running", name);
return EXIT_FALSE;
} else {
// the app exists, but it may be in a terminated state
@@ -1653,11 +1653,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
log.debug("State {}", report);
return EXIT_FALSE;
}
-log.info("Cluster {} is running:\n{}", name, report);
+log.info("Cluster {} is live:\n{}", name, report);
}
} else {
- log.info("Cluster {} exists but is not running", name);
-
+ log.info("Cluster {} exists", name);
}
return EXIT_SUCCESS;
}
[06/17] git commit: Merge commit '7c95d5b059045cc01fa9756308f50e01644abb2d' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge commit '7c95d5b059045cc01fa9756308f50e01644abb2d' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4eeed33a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4eeed33a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4eeed33a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4eeed33abd8623c094ea771e1d6cd2875c15c1fd Parents: f6bbb8b 7c95d5b Author: Steve Loughran Authored: Thu Sep 11 10:37:11 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 11 10:37:11 2014 +0100 -- .../java/org/apache/slider/client/SliderClient.java| 13 ++--- .../org/apache/slider/common/tools/SliderUtils.java| 4 +++- .../abstracttests/AbstractTestBuildSetup.groovy| 3 +++ 3 files changed, 12 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eeed33a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java --
[05/17] git commit: Merge branch 'develop' into feature/slider_install_package
Merge branch 'develop' into feature/slider_install_package Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/57bce09b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/57bce09b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/57bce09b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 57bce09b0f5c45a7e331418216b488a961398d00 Parents: fc3574e fa7130d Author: Sumit Mohanty Authored: Wed Sep 10 19:54:56 2014 -0700 Committer: Sumit Mohanty Committed: Wed Sep 10 19:54:56 2014 -0700 -- app-packages/accumulo/README.txt| 17 ++- app-packages/accumulo/pom.xml | 90 +-- app-packages/accumulo/src/assembly/accumulo.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 7 ++ .../accumulo/AccumuloMonitorSSLIT.groovy| 7 +- .../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +- .../accumulo/src/test/resources/appConfig.json | 55 ++ .../src/test/resources/appConfig_kerberos.json | 8 +- .../test/resources/appConfig_monitor_ssl.json | 4 +- .../appConfig_monitor_ssl_kerberos.json | 57 ++ .../src/test/resources/appConfig_ssl.json | 4 +- .../test/resources/appConfig_ssl_kerberos.json | 59 ++ .../command-logger/application-pkg/pom.xml | 38 --- app-packages/command-logger/slider-pkg/pom.xml | 38 --- app-packages/hbase/metainfo.xml | 2 +- pom.xml | 67 +++- slider-agent/pom.xml| 62 ++- slider-assembly/src/main/scripts/slider.py | 109 +-- slider-core/pom.xml | 102 + .../org/apache/slider/client/SliderClient.java | 51 ++--- .../apache/slider/common/tools/SliderUtils.java | 4 +- .../slider/core/launch/AppMasterLauncher.java | 6 +- .../security/FsDelegationTokenManager.java | 39 ++- .../TestFsDelegationTokenManager.groovy | 3 +- slider-funtest/pom.xml | 48 .../abstracttests/AbstractTestBuildSetup.groovy | 3 + .../framework/AgentCommandTestBase.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy| 7 +- .../funtest/framework/FileUploader.groovy | 1 + .../funtest/framework/FuntestProperties.groovy | 1 + .../slider/funtest/framework/SliderShell.groovy | 85 +-- .../slider/funtest/basic/SyspropsIT.groovy | 50 + .../hbase/slider-hbase-provider/pom.xml | 49 + src/test/clusters/script.md | 2 +- 34 files changed, 795 insertions(+), 291 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57bce09b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java --
[08/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance
directory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3f742d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3f742d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3f742d1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b3f742d1b32dd8043ac9a8e1ef546e938dc832c7
Parents: 57bce09
Author: Sumit Mohanty
Authored: Thu Sep 11 07:36:43 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 11 07:36:43 2014 -0700
--
.../agent/TestAgentClientProvider.java | 53
.../agent/TestAgentClientProvider2.java | 88
2 files changed, 88 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3f742d1/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
index 1ad59e3..4cb35aa 100644
---
a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
+++
b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
@@ -18,38 +18,22 @@ package org.apache.slider.providers.agent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.tools.TestUtility;
-import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.easymock.PowerMock;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Set;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.anyObject;
-
/**
*
*/
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ProviderUtils.class)
public class TestAgentClientProvider {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentClientProvider.class);
@@ -90,41 +74,4 @@ public class TestAgentClientProvider {
Assert.assertTrue(e.getMessage().contains("Application definition must
be provided"));
}
}
-
- @Test
- public void testPrepareAMAndConfigForLaunch() throws Exception {
-AgentClientProvider provider = new AgentClientProvider(null);
-SliderFileSystem sfs = PowerMock.createMock(SliderFileSystem.class);
-FileSystem fs = PowerMock.createMock(FileSystem.class);
-Configuration serviceConf = PowerMock.createMock(Configuration.class);
-PowerMock.mockStatic(ProviderUtils.class);
-
-expect(sfs.getFileSystem()).andReturn(fs);
-expect(fs.mkdirs(anyObject(Path.class))).andReturn(true);
-expect(ProviderUtils.addAgentTar(
-anyObject(), anyObject(String.class),
anyObject(SliderFileSystem.class), anyObject(Path.class))).
-andReturn(true);
-
-AggregateConf instanceDefinition = new AggregateConf();
-ConfTree tree = new ConfTree();
-tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
-instanceDefinition.setInternal(tree);
-
-PowerMock.replay(sfs, fs, serviceConf, ProviderUtils.class);
-
-provider.prepareAMAndConfigForLaunch(
-sfs, serviceConf, null, instanceDefinition, null,
-null, null, null, null, false);
-
-
Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
-tree.global.remove(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-
-// Verify that slider-agent.tar.gz got added
-Path tempPath = new Path(".", "temp");
-provider.prepareAMAndConfigForLaunch(
-sfs, serviceConf, null, instanceDefinition, null,
-null, null, null, tempPath, false);
-PowerMock.verify(sfs, fs, ProviderUtils.class);
-
Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3f742d1/slider-core/src/test/java/org/apache/slider/provi
[04/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance
directory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fc3574e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fc3574e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fc3574e0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: fc3574e0fa4db8fdeb94f53c6cbcbfe85d373ae8
Parents: 809dc78
Author: Sumit Mohanty
Authored: Wed Sep 10 19:54:19 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Sep 10 19:54:19 2014 -0700
--
slider-assembly/pom.xml | 20
slider-assembly/src/assembly/slider-bin.xml | 8 +++
slider-core/pom.xml | 28 +++
.../org/apache/slider/api/InternalKeys.java | 6 ++-
.../org/apache/slider/client/SliderClient.java | 12 ++---
.../org/apache/slider/common/SliderKeys.java| 1 +
.../apache/slider/common/tools/SliderUtils.java | 4 ++
.../slider/core/build/InstanceBuilder.java | 28 +--
.../apache/slider/providers/ProviderUtils.java | 33 ++--
.../providers/agent/AgentClientProvider.java| 41 +++
.../providers/agent/AgentProviderService.java | 19 ++-
.../providers/agent/TestBuildBasicAgent.groovy | 19 ---
.../agent/TestAgentClientProvider.java | 53
13 files changed, 203 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-assembly/pom.xml
--
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index c2d0ecb..ac395fb 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -104,6 +104,26 @@
+
+
+copy-to-lib
+package
+
+ copy
+
+
+
+
+ org.apache.slider
+ slider-agent
+ tar.gz
+ false
+
${project.build.directory}/lib
+ slider-agent.tar.gz
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-assembly/src/assembly/slider-bin.xml
--
diff --git a/slider-assembly/src/assembly/slider-bin.xml
b/slider-assembly/src/assembly/slider-bin.xml
index 6f3b021..c5611f8 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -59,6 +59,14 @@
${project.build.directory}/agent
+ lib
+
+slider-agent.tar.gz
+
+
+
+
+ ${project.build.directory}/agent
agent
slider-agent.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/pom.xml
--
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 326af88..96b0f83 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -484,6 +484,34 @@
jetty-sslengine
+
+ org.powermock
+ powermock-core
+ 1.5
+ test
+
+
+
+ org.powermock
+ powermock-reflect
+ 1.5
+ test
+
+
+
+ org.powermock
+ powermock-api-easymock
+ 1.5
+ test
+
+
+
+ org.powermock
+ powermock-module-junit4
+ 1.5
+ test
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index b58f1c6..ffa3600 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -47,7 +47,11 @@ public interface InternalKeys {
/**
* internal temp directory: {@value}
*/
- String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
+ String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir";
+ /**
+ * internal temp directory: {@value}
+ */
+ String INTERNAL_TMP_DIR = "internal.tmp.dir";
/**
* where a snapshot of the original conf dir is: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fc3574e0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff -
[01/17] git commit: SLIDER-415 fix funtests against secure cluster
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry f6bbb8b85 ->
f67b5507b
SLIDER-415 fix funtests against secure cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8b32aa17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8b32aa17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8b32aa17
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8b32aa17a4d9fb237d0418f47c77bfd5dc529dbf
Parents: 4a0b783
Author: Jon Maron
Authored: Wed Sep 10 10:01:10 2014 -0400
Committer: Jon Maron
Committed: Wed Sep 10 10:01:10 2014 -0400
--
.../main/java/org/apache/slider/common/tools/SliderUtils.java| 4 +++-
.../slider/funtest/abstracttests/AbstractTestBuildSetup.groovy | 3 +++
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b32aa17/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 4cdc580..4467849 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1019,7 +1019,9 @@ public final class SliderUtils {
if (authUser.getAuthenticationMethod() ==
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with
the wrong authentication method",
+ " -security has already been set up with
the wrong authentication method. "
+ + "This can occur if a file system has
already been created prior to the loading of "
+ + "the security configuration.",
authUser);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b32aa17/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
index e0b87f7..104de58 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
@@ -160,6 +160,9 @@ abstract class AbstractTestBuildSetup extends
SliderTestUtils implements Funtest
@Test
public void testSecuritySettingsValid() throws Throwable {
Configuration conf = loadSliderConf();
+if (SliderUtils.isHadoopClusterSecure(conf)) {
+ UserGroupInformation.setLoginUser(null)
+}
if (SliderUtils.maybeInitSecurity(conf)) {
log.info("Security enabled")
SliderUtils.forceLogin()
[14/17] git commit: SLIDER-422. Automatically upload slider-agent tarball to the app instance directory
SLIDER-422. Automatically upload slider-agent tarball to the app instance
directory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/219e7f8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/219e7f8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/219e7f8e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 219e7f8e3f2888209934913eaa26972ec446bc4f
Parents: db67363
Author: Sumit Mohanty
Authored: Thu Sep 11 12:45:16 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 11 12:45:16 2014 -0700
--
.../providers/agent/TestAgentProviderService.java | 18 +++---
1 file changed, 11 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/219e7f8e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
--
diff --git
a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index aa9dd60..b652bb5 100644
---
a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++
b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -272,9 +272,10 @@ public class TestAgentProviderService {
MapOperations resourceComponent = new MapOperations();
MapOperations appComponent = new MapOperations();
Path containerTmpDirPath = new Path(".", "test");
-FileSystem mockFs = new MockFileSystem();
+FileSystem mockFs = createNiceMock(FileSystem.class);
+expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
expect(sliderFileSystem.getFileSystem())
-.andReturn(new FilterFileSystem(mockFs)).anyTimes();
+.andReturn(mockFs).anyTimes();
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();
@@ -331,7 +332,7 @@ public class TestAgentProviderService {
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
-replay(access, ctx, container, sliderFileSystem);
+replay(access, ctx, container, sliderFileSystem, mockFs);
try {
mockAps.buildContainerLaunchContext(launcher,
@@ -423,7 +424,9 @@ public class TestAgentProviderService {
AgentProviderService mockAps = Mockito.spy(aps);
doReturn(access).when(mockAps).getAmState();
-doReturn("scripts/hbase_master.py").when(mockAps)
+CommandScript cs = new CommandScript();
+cs.setScript("scripts/hbase_master.py");
+doReturn(cs).when(mockAps)
.getScriptPathFromMetainfo(anyString());
Metainfo metainfo = new Metainfo();
Application application = new Application();
@@ -922,9 +925,10 @@ public class TestAgentProviderService {
MapOperations resourceComponent = new MapOperations();
MapOperations appComponent = new MapOperations();
Path containerTmpDirPath = new Path(".", "test");
-FileSystem mockFs = new MockFileSystem();
+FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
expect(sliderFileSystem.getFileSystem())
-.andReturn(new FilterFileSystem(mockFs)).anyTimes();
+.andReturn(mockFs).anyTimes();
+expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();
@@ -982,7 +986,7 @@ public class TestAgentProviderService {
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
-replay(access, ctx, container, sliderFileSystem);
+replay(access, ctx, container, sliderFileSystem, mockFs);
// build two containers
try {
[17/17] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f67b5507
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f67b5507
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f67b5507
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f67b5507b7230a025e7c54064e2614258f009116
Parents: 6e9380c a8dfc67
Author: Steve Loughran
Authored: Fri Sep 12 09:59:58 2014 +0100
Committer: Steve Loughran
Committed: Fri Sep 12 09:59:58 2014 +0100
--
app-packages/accumulo/pom.xml | 52 +++--
.../command-logger/slider-pkg/appConfig.json| 2 +-
.../command-logger/slider-pkg/metainfo.xml | 2 +-
app-packages/hbase/metainfo.xml | 2 +-
slider-assembly/pom.xml | 20 ++
slider-assembly/src/assembly/slider-bin.xml | 8 +
slider-core/pom.xml | 28 +++
.../org/apache/slider/api/InternalKeys.java | 6 +-
.../org/apache/slider/client/SliderClient.java | 12 +-
.../org/apache/slider/common/SliderKeys.java| 1 +
.../apache/slider/common/tools/SliderUtils.java | 4 +
.../slider/core/build/InstanceBuilder.java | 28 +--
.../apache/slider/providers/ProviderUtils.java | 33 ++-
.../providers/agent/AgentClientProvider.java| 41 ++--
.../providers/agent/AgentProviderService.java | 19 +-
.../web/rest/agent/ExecutionCommand.java| 16 ++
.../appmaster/web/rest/agent/StatusCommand.java | 13 ++
.../test_command_log/appConfig.json | 2 +-
.../providers/agent/TestBuildBasicAgent.groovy | 19 --
.../agent/TestAgentClientProvider2.java | 88 +++
.../agent/TestAgentProviderService.java | 230 ++-
slider-funtest/pom.xml | 7 +-
.../org/apache/chaos/remote/BuildPidPath.groovy | 54 +
.../org/apache/chaos/remote/Clustat.groovy | 64 ++
.../apache/chaos/remote/JschToCommonsLog.groovy | 78 +++
.../chaos/remote/RemoteDaemonOperations.groovy | 60 +
.../org/apache/chaos/remote/RemoteServer.groovy | 188 +++
.../org/apache/chaos/remote/SshCommands.groovy | 190 +++
.../funtest/lifecycle/AMFailuresIT.groovy | 191 +++
29 files changed, 1353 insertions(+), 105 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/pom.xml
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e10eb50,b2d10f6..baa413e
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -1186,10 -1157,12 +1181,12 @@@ public class SliderClient extends Abstr
clusterDirectory,
generatedConfDirPath,
clusterSecure
-);
+ );
- // now add the image if it was set
+ // TODO: consider supporting apps that don't have an image path
+ Path imagePath =
+ SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
[10/17] git commit: SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
SLIDER-280. Add unit and functional tests for agents discovering AM after
restart (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f1c1f705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f1c1f705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f1c1f705
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f1c1f705fe24af7e768fcf706370d0f0bb625f11
Parents: a509561
Author: Sumit Mohanty
Authored: Thu Sep 11 10:18:23 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 11 10:40:00 2014 -0700
--
.../command-logger/slider-pkg/appConfig.json| 2 +-
.../command-logger/slider-pkg/metainfo.xml | 2 +-
.../test_command_log/appConfig.json | 2 +-
slider-funtest/pom.xml | 7 +-
slider-funtest/pom.xml.orig | 357 +++
.../org/apache/chaos/remote/BuildPidPath.groovy | 54 +++
.../org/apache/chaos/remote/Clustat.groovy | 64
.../apache/chaos/remote/JschToCommonsLog.groovy | 78
.../chaos/remote/RemoteDaemonOperations.groovy | 60
.../org/apache/chaos/remote/RemoteServer.groovy | 188 ++
.../org/apache/chaos/remote/SshCommands.groovy | 190 ++
.../funtest/lifecycle/AMFailuresIT.groovy | 191 ++
12 files changed, 1191 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/appConfig.json
--
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json
b/app-packages/command-logger/slider-pkg/appConfig.json
index bfdffb8..791043f 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -11,7 +11,7 @@
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/command-logger",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
-"site.cl-site.logfile.location":
"${AGENT_LOG_ROOT}/app/log/operations.log",
+"site.cl-site.logfile.location":
"${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/metainfo.xml
--
diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml
b/app-packages/command-logger/slider-pkg/metainfo.xml
index 976fa81..b8de9a3 100644
--- a/app-packages/command-logger/slider-pkg/metainfo.xml
+++ b/app-packages/command-logger/slider-pkg/metainfo.xml
@@ -42,7 +42,7 @@
tarball
-files/command_log.tar
+files/command-logger.tar
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-core/src/test/app_packages/test_command_log/appConfig.json
--
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json
b/slider-core/src/test/app_packages/test_command_log/appConfig.json
index 684924d..7586a0c 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json
@@ -12,7 +12,7 @@
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/command-logger",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
-"site.cl-site.logfile.location":
"${AGENT_LOG_ROOT}/app/log/operations.log",
+"site.cl-site.logfile.location":
"${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/pom.xml
--
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 124251f..55b6351 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -121,6 +121,7 @@
apache-slider-command-logger.zip
../slider-core/src/test/app_packages/test_command_log/resources.json
../slider-core/src/test/app_packages/test_command_log/appConfig.json
+
/Users/gsaha/work/workspace/ambari-vagrant/centos6.4
@@ -288,7 +289,11 @@
org.codehaus
[03/17] git commit: Remove -bin from hbase tar ball name
Remove -bin from hbase tar ball name
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa7130d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa7130d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa7130d3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: fa7130d306dfba1e60a70f57bb1735eb77424872
Parents: 7c95d5b
Author: tedyu
Authored: Wed Sep 10 12:52:18 2014 -0700
Committer: tedyu
Committed: Wed Sep 10 12:52:18 2014 -0700
--
app-packages/hbase/metainfo.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa7130d3/app-packages/hbase/metainfo.xml
--
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index 10bee97..1072a52 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -160,7 +160,7 @@
tarball
-files/hbase-${pkg.version}-bin.tar.gz
+files/hbase-${pkg.version}.tar.gz
[02/17] git commit: SLIDER-381 Slider-client exists command does not report status correctly
SLIDER-381 Slider-client exists command does not report status correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7c95d5b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7c95d5b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7c95d5b0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7c95d5b059045cc01fa9756308f50e01644abb2d
Parents: 8b32aa1
Author: Steve Loughran
Authored: Wed Sep 10 18:04:18 2014 +0100
Committer: Steve Loughran
Committed: Wed Sep 10 18:04:18 2014 +0100
--
.../java/org/apache/slider/client/SliderClient.java| 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7c95d5b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef0e63c..7c1fd1e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1623,10 +1623,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @return exit code
*/
@VisibleForTesting
- public int actionExists(String name, boolean live) throws YarnException,
IOException {
+ public int actionExists(String name, boolean checkLive) throws
YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(name);
-log.debug("actionExists({}, {})", name, live);
+log.debug("actionExists({}, {})", name, checkLive);
//initial probe for a cluster in the filesystem
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
@@ -1636,10 +1636,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
//test for liveness if desired
-if (live) {
+if (checkLive) {
ApplicationReport instance = findInstance(name);
if (instance == null) {
-log.info("cluster {} not running", name);
+log.info("Cluster {} not running", name);
return EXIT_FALSE;
} else {
// the app exists, but it may be in a terminated state
@@ -1653,11 +1653,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
log.debug("State {}", report);
return EXIT_FALSE;
}
-log.info("Cluster {} is running:\n{}", name, report);
+log.info("Cluster {} is live:\n{}", name, report);
}
} else {
- log.info("Cluster {} exists but is not running", name);
-
+ log.info("Cluster {} exists", name);
}
return EXIT_SUCCESS;
}
