git commit: javadocs on SliderUtils.appendToUrl
Repository: incubator-slider
Updated Branches:
refs/heads/develop 3fecc58c9 -> 572effbb9
javadocs on SliderUtils.appendToUrl
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/572effbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/572effbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/572effbb
Branch: refs/heads/develop
Commit: 572effbb962c911be858e9c2c331d742e5f28a55
Parents: 3fecc58
Author: Steve Loughran
Authored: Wed Jul 30 13:51:06 2014 +0100
Committer: Steve Loughran
Committed: Wed Jul 30 13:51:06 2014 +0100
--
.../main/java/org/apache/slider/common/tools/SliderUtils.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/572effbb/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 17f8b70..8e412ee 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1357,9 +1357,9 @@ public final class SliderUtils {
/**
* Append a list of paths, inserting "/" signs as appropriate
- * @param base
- * @param paths
- * @return
+ * @param base base path/URL
+ * @param paths subpaths
+ * @return base+"/"+paths[0]+"/"+paths[1]...
*/
public static String appendToURL(String base, String...paths) {
String result = base;
[2/2] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/120c3b9c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/120c3b9c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/120c3b9c Branch: refs/heads/develop Commit: 120c3b9c4684ecc8233a40f544e5ef8325440614 Parents: 54eec9d 2ee8374 Author: Steve Loughran Authored: Fri Aug 1 15:29:40 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 1 15:29:40 2014 +0100 -- .../org/apache/slider/providers/agent/AgentProviderService.java | 4 +++- .../org/apache/slider/providers/hbase/HBaseClientProvider.java | 2 ++ .../main/java/org/apache/slider/providers/hbase/HBaseKeys.java | 4 .../org/apache/slider/providers/hbase/HBaseProviderService.java | 5 + .../main/java/org/apache/slider/providers/hbase/HBaseRoles.java | 3 +++ 5 files changed, 17 insertions(+), 1 deletion(-) --
[1/2] git commit: SliderLauncher: javadocs, unchecked warning suppression, and logging
Repository: incubator-slider
Updated Branches:
refs/heads/develop 2ee8374eb -> 120c3b9c4
SliderLauncher: javadocs, unchecked warning suppression, and logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/54eec9d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/54eec9d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/54eec9d4
Branch: refs/heads/develop
Commit: 54eec9d47948b65e4be5bdfb0d72676ed806faec
Parents: 572effb
Author: Steve Loughran
Authored: Fri Aug 1 15:29:24 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 15:29:24 2014 +0100
--
.../java/org/apache/slider/core/main/ServiceLauncher.java | 7 ++-
1 file changed, 2 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/54eec9d4/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index c92dfda..846aca1 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -202,15 +202,11 @@ public class ServiceLauncher
* to the configuration, and service to the service.
*
* @param conf configuration to use
- * @throws ClassNotFoundException no such class
- * @throws InstantiationException no empty constructor,
- * problems with dependencies
* @throws ClassNotFoundException classname not on the classpath
* @throws IllegalAccessException not allowed at the class
* @throws InstantiationException not allowed to instantiate it
- * @throws InterruptedException thread interrupted
- * @throws Throwable any other failure
*/
+ @SuppressWarnings("unchecked")
public Service instantiateService(Configuration conf)
throws ClassNotFoundException, InstantiationException,
IllegalAccessException,
ExitUtil.ExitException, NoSuchMethodException, InvocationTargetException
{
@@ -447,6 +443,7 @@ public class ServiceLauncher
try {
conf.addResource(file.toURI().toURL());
} catch (MalformedURLException e) {
+ LOG.debug("File {} cannot be converted to URL: {}", e);
exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
ARG_CONF + ": configuration file path invalid: " + file);
}
[1/3] git commit: fix indentation in POM
Repository: incubator-slider
Updated Branches:
refs/heads/develop 120c3b9c4 -> 8c6c6b67c
fix indentation in POM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/463b0257
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/463b0257
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/463b0257
Branch: refs/heads/develop
Commit: 463b025775c2ec3fe16faccc47e6542e9e922258
Parents: 120c3b9
Author: Steve Loughran
Authored: Fri Aug 1 19:56:52 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 19:56:52 2014 +0100
--
slider-assembly/pom.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/463b0257/slider-assembly/pom.xml
--
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index cfe970c..d5299ee 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -308,9 +308,9 @@
${project.build.directory}/agent
-
-slider-agent.tar.gz
-
+
+slider-agent.tar.gz
+
[2/3] git commit: SLIDER-149 make app type a valid hostname pattern
SLIDER-149 make app type a valid hostname pattern
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/51980467
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/51980467
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/51980467
Branch: refs/heads/develop
Commit: 51980467d8a60ebd5bdc90bf6e30cdab8781920e
Parents: 463b025
Author: Steve Loughran
Authored: Fri Aug 1 19:57:18 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 19:57:18 2014 +0100
--
slider-core/src/main/java/org/apache/slider/common/SliderKeys.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51980467/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 0ba562a..c85ad91 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -50,7 +50,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
/**
* Application type for YARN {@value}
*/
- String APP_TYPE = "org.apache.slider";
+ String APP_TYPE = "org-apache-slider";
/**
* JVM arg to force IPv4 {@value}
[3/3] git commit: SLIDER-149 rework all tests to dynamically generate valid cluster names from the test methods -expect in specific cases where explicit names are to be chosen
SLIDER-149 rework all tests to dynamically generate valid cluster names from the test methods -expect in specific cases where explicit names are to be chosen Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8c6c6b67 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8c6c6b67 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8c6c6b67 Branch: refs/heads/develop Commit: 8c6c6b67c8339186624f09bdbbc1223531dbaed5 Parents: 5198046 Author: Steve Loughran Authored: Fri Aug 1 20:45:09 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 1 20:45:09 2014 +0100 -- .../agent/actions/TestActionExists.groovy | 2 +- .../slider/agent/actions/TestActionList.groovy | 2 +- .../agent/actions/TestActionStatus.groovy | 6 +-- .../agent/freezethaw/TestFreezeCommands.groovy | 3 +- .../TestFreezeThawMasterlessAM.groovy | 3 +- .../freezethaw/TestFreezeUnknownCluster.groovy | 3 +- .../freezethaw/TestThawUnknownCluster.groovy| 3 +- .../agent/standalone/TestBadAMHeap.groovy | 3 +- .../agent/standalone/TestBadYarnQueue.groovy| 3 +- .../standalone/TestBuildStandaloneAM.groovy | 6 +-- .../standalone/TestDestroyMasterlessAM.groovy | 3 +- .../standalone/TestKillMasterlessAM.groovy | 3 +- .../standalone/TestStandaloneAgentAM.groovy | 3 +- .../standalone/TestStandaloneRegistryAM.groovy | 3 +- .../agent/TestAgentAMManagementWS.groovy| 4 +- .../slider/providers/agent/TestAgentEcho.groovy | 4 +- .../providers/agent/TestBuildBasicAgent.groovy | 39 .../curator/TestRegistryRestResources.groovy| 10 +++-- .../publisher/TestPublisherRestResources.groovy | 5 +-- .../apache/slider/test/SliderTestBase.groovy| 17 - .../slider/test/YarnMiniClusterTestBase.groovy | 21 +-- .../test/YarnZKMiniClusterTestBase.groovy | 22 +-- .../providers/accumulo/AccumuloTestBase.groovy | 2 +- .../live/TestAccCorrectInstanceName.groovy | 4 +- .../live/TestAccFlexTablets133Mgr113.groovy | 3 +- .../accumulo/live/TestAccFlexTablets1to3.groovy | 2 +- .../accumulo/live/TestAccFreezeThaw.groovy | 4 +- .../accumulo/live/TestAccLiveHDFSArchive.groovy | 4 +- .../live/TestAccLiveLocalArchive.groovy | 4 +- .../accumulo/live/TestAccM1T1GC1Mon1.groovy | 3 +- .../accumulo/live/TestAccM2T2GC1Mon1.groovy | 4 +- .../accumulo/live/TestAccumuloAMWebApp.groovy | 4 +- .../live/TestInvalidMonitorAddress.groovy | 3 +- .../TestFreezeThawClusterFromArchive.groovy | 3 +- .../archives/TestLiveClusterFromArchive.groovy | 4 +- .../TestLiveClusterFromArchiveOnHDFS.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 3 +- .../failures/TestFailedRegionService.groovy | 7 ++-- .../failures/TestFailureThreshold.groovy| 5 +-- .../failures/TestKilledHBaseAM.groovy | 3 +- .../failures/TestKilledHBaseMaster.groovy | 4 +- .../flexing/TestClusterFlex0To1.groovy | 2 +- .../flexing/TestClusterFlex1To1.groovy | 4 +- .../flexing/TestClusterFlex1To2.groovy | 4 +- .../flexing/TestClusterFlex2DownTo1.groovy | 2 +- .../flexing/TestClusterFlex2To5.groovy | 4 +- .../flexing/TestClusterFlexDownToZero.groovy| 4 +- .../flexing/TestHMasterFlex1To2.groovy | 4 +- ...reezeReconfigureThawLiveRegionService.groovy | 3 +- .../TestFreezeThawLiveRegionService.groovy | 3 +- .../minicluster/live/Test2Master2RS.groovy | 3 +- .../minicluster/live/TestHBaseMaster.groovy | 3 +- .../live/TestHBaseMasterOnHDFS.groovy | 4 +- .../live/TestHBaseMasterWithBadHeap.groovy | 3 +- .../live/TestLiveRegionServiceOnHDFS.groovy | 4 +- .../live/TestLiveTwoNodeRegionService.groovy| 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 3 +- .../masterless/TestRoleOptPropagation.groovy| 6 +-- .../TestSliderConfDirToMasterlessAM.groovy | 3 +- 59 files changed, 142 insertions(+), 157 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8c6c6b67/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy -- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy index cb05fd0..ee8baff 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy @@ -45,7 +45,7 @@ class TestActionExists extends AgentMiniClusterTestBa
[2/2] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/067c3060 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/067c3060 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/067c3060 Branch: refs/heads/develop Commit: 067c3060a828b9d663cdcfe0c344ea4e6693ece3 Parents: e239bd3 499181d Author: Steve Loughran Authored: Sat Aug 2 10:31:59 2014 +0100 Committer: Steve Loughran Committed: Sat Aug 2 10:31:59 2014 +0100 -- slider-agent/src/test/python/unitTests.py| 3 ++- .../hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy | 2 +- .../providers/hbase/minicluster/live/TestTwoLiveClusters.groovy | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) --
[1/2] git commit: SLIDER-266: checks for null keys and vals setting env vars and JVM properties on launched code
Repository: incubator-slider
Updated Branches:
refs/heads/develop 499181d95 -> 067c3060a
SLIDER-266: checks for null keys and vals setting env vars and JVM properties
on launched code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e239bd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e239bd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e239bd3c
Branch: refs/heads/develop
Commit: e239bd3cbc6eacf64b72bbb08008d66a73221ffa
Parents: 8c6c6b6
Author: Steve Loughran
Authored: Sat Aug 2 10:31:49 2014 +0100
Committer: Steve Loughran
Committed: Sat Aug 2 10:31:49 2014 +0100
--
.../main/java/org/apache/slider/core/launch/AbstractLauncher.java | 3 +++
.../org/apache/slider/core/launch/JavaCommandLineBuilder.java | 3 +++
2 files changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e239bd3c/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index d8c3522..8652b4c 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.launch;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -238,6 +239,8 @@ public abstract class AbstractLauncher extends Configured {
setEnv("CLASSPATH", classpath.buildClasspath());
}
public void setEnv(String var, String value) {
+Preconditions.checkArgument(var != null, "null variable name");
+Preconditions.checkArgument(value != null, "null value");
env.put(var, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e239bd3c/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 7b60461..0367e06 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -19,6 +19,7 @@
package org.apache.slider.core.launch;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
@@ -63,6 +64,8 @@ public class JavaCommandLineBuilder extends
CommandLineBuilder {
* @param value
*/
public void sysprop(String property, String value) {
+Preconditions.checkArgument(property != null, "null property name");
+Preconditions.checkArgument(value != null, "null value");
add("-D" + property + "=" + value);
}
[02/11] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/edd96e60 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/edd96e60 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/edd96e60 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: edd96e600588c3bc30bed6a49d42f31119154441 Parents: cfc6e1d a613e20 Author: Steve Loughran Authored: Mon Jul 7 15:30:00 2014 +0100 Committer: Steve Loughran Committed: Mon Jul 7 15:30:00 2014 +0100 -- app-packages/accumulo/appConfig.json| 4 +- app-packages/accumulo/pom.xml | 107 - app-packages/accumulo/resources.json| 15 +- app-packages/accumulo/src/assembly/accumulo.xml | 1 + .../AccumuloAgentCommandTestBase.groovy | 52 .../funtest/accumulo/AccumuloBasicIT.groovy | 117 + .../accumulo/AccumuloMonitorSSLIT.groovy| 73 ++ .../funtest/accumulo/AccumuloReadWriteIT.groovy | 116 + .../StubToForceGroovyTestsToCompile.java| 22 ++ .../test/resources/appConfig_monitor_ssl.json | 62 + .../accumulo/src/test/resources/resources.json | 36 +++ app-packages/hbase/README.txt | 3 + pom.xml | 2 +- .../apache/slider/common/SliderXmlConfKeys.java | 2 + .../slider/core/main/ServiceLauncher.java | 109 +++-- .../server/appmaster/SliderAppMaster.java | 46 +++- .../slider/server/appmaster/state/AppState.java | 2 +- .../server/appmaster/web/SliderAMWebApp.java| 2 +- .../server/appmaster/web/SliderAmIpFilter.java | 1 + .../server/services/curator/CuratorService.java | 2 +- .../resources/webapps/static/yarn.dt.plugins.js | 4 +- .../slider/test/YarnMiniClusterTestBase.groovy | 2 +- slider-funtest/pom.xml | 5 +- .../framework/AgentCommandTestBase.groovy | 237 +++ .../accumulo/StubToForceGroovySrcToCompile.java | 22 ++ .../basic/TestClusterConnectivity.groovy| 35 ++- .../lifecycle/AgentCommandTestBase.groovy | 233 -- .../lifecycle/TestAgentClusterLifecycle.groovy | 1 + .../funtest/lifecycle/TestAgentFailures.groovy | 1 + .../funtest/lifecycle/TestAgentFailures2.groovy | 1 + .../lifecycle/TestAppsThroughAgent.groovy | 1 + .../lifecycle/TestClusterBuildDestroy.groovy| 1 + .../accumulo/AccumuloProviderService.java | 4 + 33 files changed, 1025 insertions(+), 296 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/edd96e60/pom.xml --
[03/11] git commit: SLIDER-149 javadocs and whitespace
SLIDER-149 javadocs and whitespace
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9ec12883
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9ec12883
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9ec12883
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 9ec1288318776332cee9544411a50706700f305c
Parents: edd96e6
Author: Steve Loughran
Authored: Wed Jul 9 12:02:08 2014 +0100
Committer: Steve Loughran
Committed: Wed Jul 9 12:02:08 2014 +0100
--
.../src/main/java/org/apache/slider/core/persist/JsonSerDeser.java | 2 +-
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ec12883/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 3512168..792dcdf 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -197,7 +197,7 @@ public class JsonSerDeser {
/**
* Convert an object to a JSON string
- * @param o object to convert
+ * @param instance instance to convert
* @return a JSON string description
* @throws JsonParseException parse problems
* @throws JsonMappingException O/J mapping problems
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ec12883/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 2c1b270..47a67ac 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -129,6 +129,7 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
KEY_TEST_TIMEOUT,
DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
)
+
@BeforeClass
public static void checkWindowsSupport() {
if (Shell.WINDOWS) {
[09/11] git commit: SLIDER-149 mark some CuratorService methods as final
SLIDER-149 mark some CuratorService methods as final
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c2dadcb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c2dadcb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c2dadcb7
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c2dadcb7051b52089ebc927ff4e30befeadb44e3
Parents: e7607a6
Author: Steve Loughran
Authored: Sat Aug 2 13:49:36 2014 +0100
Committer: Steve Loughran
Committed: Sat Aug 2 13:49:36 2014 +0100
--
.../slider/server/services/curator/CuratorService.java | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c2dadcb7/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
index 645bc8f..b5b0c14 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
@@ -65,11 +65,11 @@ public class CuratorService extends AbstractService {
closeCuratorComponent(curator);
}
- public CuratorFramework getCurator() {
+ public final CuratorFramework getCurator() {
return curator;
}
- protected void closeCuratorComponent(Closeable closeable) {
+ protected final void closeCuratorComponent(Closeable closeable) {
try {
IOUtils.closeStream(closeable);
} catch (Throwable ignored) {
@@ -78,21 +78,21 @@ public class CuratorService extends AbstractService {
}
}
- public String pathForServicetype(String servicetype) {
+ public final String pathForServicetype(String servicetype) {
return ZKPaths.makePath(getBasePath(), servicetype);
}
- protected String pathForInstance(String servicetype, String id) {
+ protected final String pathForInstance(String servicetype, String id) {
Preconditions.checkNotNull(servicetype);
Preconditions.checkNotNull(id);
return ZKPaths.makePath(pathForServicetype(servicetype), id);
}
- public String getBasePath() {
+ public final String getBasePath() {
return basePath;
}
- public CuratorHelper getCuratorHelper() {
+ public final CuratorHelper getCuratorHelper() {
return curatorHelper;
}
}
[11/11] git commit: SLIDER-149 YARN and slider registries living side by side, though YARN reg is doing simple registration only
SLIDER-149 YARN and slider registries living side by side, though YARN reg is
doing simple registration only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e1c77f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e1c77f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e1c77f8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7e1c77f891fa797f12b282417c1bb8b21d6c4e18
Parents: 96f4acb
Author: Steve Loughran
Authored: Mon Aug 4 14:03:02 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 4 14:03:02 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 26 ++--
.../server/appmaster/SliderAppMaster.java | 2 ++
.../server/services/curator/CuratorHelper.java | 5
.../server/services/curator/CuratorService.java | 17 +
.../utility/AbstractSliderLaunchedService.java | 6 ++---
.../standalone/TestStandaloneRegistryAM.groovy | 5
6 files changed, 56 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 6b67342..87af625 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -34,6 +34,7 @@ import
org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper;
import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -666,7 +667,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
@@ -1233,7 +1234,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
*/
private boolean getUsingMiniMRCluster() {
return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
- false);
+false);
}
/**
@@ -2334,6 +2335,16 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return getRegistry().listInstances(SliderKeys.APP_TYPE);
}
+
+ /**
+ * Get an on-demand path jumper
+ * @return a class that can dump the contents of the registry
+ */
+ @VisibleForTesting
+ public ZKPathDumper dumpSliderRegistry() throws SliderException, IOException
{
+return getRegistry().dumpPath();
+ }
+
/**
* List instances in the registry
* @return the instance IDs
@@ -2410,6 +2421,17 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return maybeStartYarnRegistry();
}
+
+ /**
+ * Get an on-demand path jumper
+ * @return a class that can dump the contents of the registry
+ */
+ @VisibleForTesting
+ public ZKPathDumper dumpYarnRegistry() throws SliderException, IOException {
+return getYarnRegistry().dumpPath();
+ }
+
+
/**
* Output to standard out/stderr (implementation specific detail)
* @param src source
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 87c5295..236c5b6 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -603,6 +603,8 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
//registry
log.info("Starting slider registry");
sliderRegistry = startRegistrationService();
+ log.info(sliderRegistry.toString());
+
log.info("Starting Yarn registry");
yarnRegistry =
[01/11] git commit: SLIDER-149: service registry is on local hadoop trunk builds only
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry [created] 7e1c77f89 SLIDER-149: service registry is on local hadoop trunk builds only Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cfc6e1d0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cfc6e1d0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cfc6e1d0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: cfc6e1d062d7478aa9b48b3b1c88a611a5c0ab44 Parents: 7823d3a Author: Steve Loughran Authored: Thu Jul 3 09:09:26 2014 +0100 Committer: Steve Loughran Committed: Thu Jul 3 09:09:26 2014 +0100 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cfc6e1d0/pom.xml -- diff --git a/pom.xml b/pom.xml index 0076561..2f84d29 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ -2.4.0 +3.0.0-SNAPSHOT 0.98.3-hadoop2 1.6.0
[05/11] git commit: SLIDER-149 yarn-registry dependency
SLIDER-149 yarn-registry dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/adc4640e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/adc4640e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/adc4640e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: adc4640e5154964efe6948bf6788123ea9c294e0
Parents: 4058b9a
Author: Steve Loughran
Authored: Fri Aug 1 19:52:39 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 19:52:39 2014 +0100
--
pom.xml | 6 ++
slider-core/pom.xml | 6 ++
2 files changed, 12 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/adc4640e/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 0509964..b2670b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -518,6 +518,12 @@
org.apache.hadoop
+hadoop-yarn-registry
+${hadoop.version}
+
+
+
+org.apache.hadoop
hadoop-yarn-server-web-proxy
${hadoop.version}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/adc4640e/slider-core/pom.xml
--
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index a44e3bc..19e539e 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -306,6 +306,12 @@
org.apache.hadoop
+ hadoop-yarn-registry
+ compile
+
+
+
+ org.apache.hadoop
hadoop-minicluster
test
[08/11] git commit: SLIDER-141 registry coming up on server; some test with the existing registry is now failing. Clash of roots?
SLIDER-141 registry coming up on server; some test with the existing registry
is now failing. Clash of roots?
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7607a61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7607a61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7607a61
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e7607a6179902c44c7dda2d66d7a6588e621ef71
Parents: bfbe312
Author: Steve Loughran
Authored: Fri Aug 1 21:56:30 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 21:56:30 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 87 +++-
.../org/apache/slider/common/params/ArgOps.java | 2 +-
.../core/registry/YARNRegistryClient.java | 85 ---
.../slider/core/registry/YarnAppListClient.java | 86 +++
.../providers/AbstractProviderService.java | 2 +-
.../slider/providers/ProviderService.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 19 +++--
.../server/appmaster/web/SliderAMWebApp.java| 7 +-
.../utility/AbstractSliderLaunchedService.java | 20 +++--
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 4 +-
.../standalone/TestStandaloneAgentAM.groovy | 11 ++-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 2 +-
.../publisher/TestAgentProviderService.java | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
16 files changed, 200 insertions(+), 136 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..6b67342 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.OptionKeys;
@@ -86,7 +88,7 @@ import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
-import org.apache.slider.core.registry.YARNRegistryClient;
+import org.apache.slider.core.registry.YarnAppListClient;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
@@ -154,10 +156,16 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* Yarn client service
*/
private SliderYarnClientImpl yarnClient;
- private YARNRegistryClient YARNRegistryClient;
+ private YarnAppListClient YarnAppListClient;
private AggregateConf launchedInstanceDefinition;
private SliderRegistryService registry;
+
+ /**
+ * The YARN registry service
+ */
+ private YarnRegistryService yarnRegistry;
+
/**
* Constructor
*/
@@ -196,8 +204,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
//here the superclass is inited; getConfig returns a non-null value
sliderFileSystem = new SliderFileSystem(getConfig());
-YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+YarnAppListClient =
+ new YarnAppListClient(yarnClient, getUsername(), getConfig());
}
/**
@@ -1014,8 +1022,12 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
DEFAULT_REGISTRY_PATH);
+addMandatoryConfOptionToCLI(commandLine, config,
RegistryConstants.REGISTRY_ZK_QUORUM);
addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
-
+define(commandLine, RegistryConstants.REGISTRY_ZK_QUORUM,
+getYarnRegistry().getCurrentZookeeperQuorum());
+
+
if (clusterSecure) {
// if the c
[06/11] git commit: SLIDER-149 yarn-registry alongside slider registry
SLIDER-149 yarn-registry alongside slider registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d8d3671d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d8d3671d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d8d3671d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d8d3671da22b65b67ee94ff620e39b6661722130
Parents: adc4640
Author: Steve Loughran
Authored: Fri Aug 1 19:55:15 2014 +0100
Committer: Steve Loughran
Committed: Fri Aug 1 19:55:15 2014 +0100
--
.../core/registry/info/RegisteredEndpoint.java | 41 +++---
.../providers/AbstractProviderService.java | 24
.../slider/providers/ProviderService.java | 14 +++--
.../providers/agent/AgentProviderService.java | 25 +++--
.../slideram/SliderAMProviderService.java | 9 +--
.../server/appmaster/SliderAppMaster.java | 58 ++--
.../utility/AbstractSliderLaunchedService.java | 53 +++---
.../model/mock/MockProviderService.groovy | 8 ++-
.../publisher/TestAgentProviderService.java | 7 ++-
.../providers/hbase/HBaseProviderService.java | 8 ++-
10 files changed, 155 insertions(+), 92 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
index f3477d0..8c065e2 100644
---
a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++
b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
@@ -18,6 +18,8 @@
package org.apache.slider.core.registry.info;
+import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
+import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -37,58 +39,29 @@ public class RegisteredEndpoint {
/**
* URL: {@value}
*/
- public static final String TYPE_URL = "url";
+ public static final String TYPE_URL = AddressTypes.ADDRESS_URI;
- /**
- * hostname: {@value}
- */
- public static final String TYPE_HOSTNAME = "hostname";
/**
* "hostname:port" pair: {@value}
*/
- public static final String TYPE_INETADDRESS = "inetaddress";
+ public static final String TYPE_INETADDRESS =
AddressTypes.ADDRESS_HOSTNAME_AND_PORT;
/**
* simple path string: {@value}
*/
- public static final String TYPE_PATH = "path";
+ public static final String TYPE_PATH = AddressTypes.ADDRESS_PATH;
// standard protocols
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_TCP = "tcp";
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_UDP = "udp";
-
- /**
- * HTTP: {@value}
- */
-
- public static final String PROTOCOL_HTTP = "http";
-
- /**
- * HTTPS: {@value}
- */
-
- public static final String PROTOCOL_HTTPS = "http";
-
/**
* Classic "Writable" Hadoop IPC: {@value}
*/
- public static final String PROTOCOL_HADOOP_RPC = "org.apache.hadoop.ipc.RPC";
+ public static final String PROTOCOL_HADOOP_RPC =
ProtocolTypes.PROTOCOL_HADOOP_IPC;
/**
* Protocol buffer based Hadoop IPC: {@value}
*/
- public static final String PROTOCOL_HADOOP_PROTOBUF =
"org.apache.hadoop.ipc.Protobuf";
+ public static final String PROTOCOL_HADOOP_PROTOBUF =
ProtocolTypes.PROTOCOL_HADOOP_IPC_PROTOBUF;
/**
* The address -format is driven by the type entry
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e35227c..b60328f 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,8 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org
[07/11] git commit: Merge remote-tracking branch 'apache/develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge remote-tracking branch 'apache/develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfbe3125 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfbe3125 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfbe3125 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: bfbe31253c0fd90d956adc14fef024219311ecf2 Parents: d8d3671 8c6c6b6 Author: Steve Loughran Authored: Fri Aug 1 20:46:30 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 1 20:46:30 2014 +0100 -- slider-assembly/pom.xml | 6 +-- .../org/apache/slider/common/SliderKeys.java| 2 +- .../agent/actions/TestActionExists.groovy | 2 +- .../slider/agent/actions/TestActionList.groovy | 2 +- .../agent/actions/TestActionStatus.groovy | 6 +-- .../agent/freezethaw/TestFreezeCommands.groovy | 3 +- .../TestFreezeThawMasterlessAM.groovy | 3 +- .../freezethaw/TestFreezeUnknownCluster.groovy | 3 +- .../freezethaw/TestThawUnknownCluster.groovy| 3 +- .../agent/standalone/TestBadAMHeap.groovy | 3 +- .../agent/standalone/TestBadYarnQueue.groovy| 3 +- .../standalone/TestBuildStandaloneAM.groovy | 6 +-- .../standalone/TestDestroyMasterlessAM.groovy | 3 +- .../standalone/TestKillMasterlessAM.groovy | 3 +- .../standalone/TestStandaloneAgentAM.groovy | 3 +- .../standalone/TestStandaloneRegistryAM.groovy | 3 +- .../agent/TestAgentAMManagementWS.groovy| 4 +- .../slider/providers/agent/TestAgentEcho.groovy | 4 +- .../providers/agent/TestBuildBasicAgent.groovy | 39 .../curator/TestRegistryRestResources.groovy| 10 +++-- .../publisher/TestPublisherRestResources.groovy | 5 +-- .../apache/slider/test/SliderTestBase.groovy| 17 - .../slider/test/YarnMiniClusterTestBase.groovy | 21 +-- .../test/YarnZKMiniClusterTestBase.groovy | 22 +-- .../providers/accumulo/AccumuloTestBase.groovy | 2 +- .../live/TestAccCorrectInstanceName.groovy | 4 +- .../live/TestAccFlexTablets133Mgr113.groovy | 3 +- .../accumulo/live/TestAccFlexTablets1to3.groovy | 2 +- .../accumulo/live/TestAccFreezeThaw.groovy | 4 +- .../accumulo/live/TestAccLiveHDFSArchive.groovy | 4 +- .../live/TestAccLiveLocalArchive.groovy | 4 +- .../accumulo/live/TestAccM1T1GC1Mon1.groovy | 3 +- .../accumulo/live/TestAccM2T2GC1Mon1.groovy | 4 +- .../accumulo/live/TestAccumuloAMWebApp.groovy | 4 +- .../live/TestInvalidMonitorAddress.groovy | 3 +- .../TestFreezeThawClusterFromArchive.groovy | 3 +- .../archives/TestLiveClusterFromArchive.groovy | 4 +- .../TestLiveClusterFromArchiveOnHDFS.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 3 +- .../failures/TestFailedRegionService.groovy | 7 ++-- .../failures/TestFailureThreshold.groovy| 5 +-- .../failures/TestKilledHBaseAM.groovy | 3 +- .../failures/TestKilledHBaseMaster.groovy | 4 +- .../flexing/TestClusterFlex0To1.groovy | 2 +- .../flexing/TestClusterFlex1To1.groovy | 4 +- .../flexing/TestClusterFlex1To2.groovy | 4 +- .../flexing/TestClusterFlex2DownTo1.groovy | 2 +- .../flexing/TestClusterFlex2To5.groovy | 4 +- .../flexing/TestClusterFlexDownToZero.groovy| 4 +- .../flexing/TestHMasterFlex1To2.groovy | 4 +- ...reezeReconfigureThawLiveRegionService.groovy | 3 +- .../TestFreezeThawLiveRegionService.groovy | 3 +- .../minicluster/live/Test2Master2RS.groovy | 3 +- .../minicluster/live/TestHBaseMaster.groovy | 3 +- .../live/TestHBaseMasterOnHDFS.groovy | 4 +- .../live/TestHBaseMasterWithBadHeap.groovy | 3 +- .../live/TestLiveRegionServiceOnHDFS.groovy | 4 +- .../live/TestLiveTwoNodeRegionService.groovy| 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 3 +- .../masterless/TestRoleOptPropagation.groovy| 6 +-- .../TestSliderConfDirToMasterlessAM.groovy | 3 +- 61 files changed, 146 insertions(+), 161 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbe3125/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy --
[10/11] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96f4acb0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96f4acb0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96f4acb0 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 96f4acb082eefa7affd22878f4a603e3b6702280 Parents: c2dadcb 067c306 Author: Steve Loughran Authored: Sat Aug 2 13:49:40 2014 +0100 Committer: Steve Loughran Committed: Sat Aug 2 13:49:40 2014 +0100 -- slider-agent/src/test/python/unitTests.py| 3 ++- .../java/org/apache/slider/core/launch/AbstractLauncher.java | 3 +++ .../org/apache/slider/core/launch/JavaCommandLineBuilder.java| 3 +++ .../hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy | 2 +- .../providers/hbase/minicluster/live/TestTwoLiveClusters.groovy | 4 ++-- 5 files changed, 11 insertions(+), 4 deletions(-) --
[04/11] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: pom.xml Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4058b9ab Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4058b9ab Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4058b9ab Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4058b9ab9deec38b9ecce1ef5c15eb0a4fe61022 Parents: 9ec1288 120c3b9 Author: Steve Loughran Authored: Fri Aug 1 16:47:03 2014 +0100 Committer: Steve Loughran Committed: Fri Aug 1 16:47:03 2014 +0100 -- app-packages/accumulo/appConfig.json| 3 +- app-packages/accumulo/pom.xml | 20 +- .../command-logger/application-pkg/pom.xml | 7 +- .../command-logger/slider-pkg/appConfig.json| 1 - app-packages/command-logger/slider-pkg/pom.xml | 2 +- app-packages/hbase/README.txt | 10 +- app-packages/hbase/appConfig.json | 11 +- app-packages/hbase/get-hbase-site.sh| 25 + app-packages/hbase/pom.xml | 154 - app-packages/hbase/resources.json | 8 +- app-packages/hbase/src/assembly/hbase.xml | 1 + .../hbase/HBaseAgentCommandTestBase.groovy | 52 ++ .../slider/funtest/hbase/HBaseBasicIT.groovy| 114 .../funtest/hbase/HBaseMonitorSSLIT.groovy | 73 +++ .../hbase/StubToForceGroovyTestsToCompile.java | 22 + .../test/resources/appConfig_monitor_ssl.json | 68 +++ .../hbase/src/test/resources/resources.json | 21 + app-packages/memcached-win/README.txt | 36 ++ app-packages/memcached-win/appConfig.json | 22 + app-packages/memcached-win/metainfo.xml | 51 ++ .../jmemcached-cli-1.0.0.jar.REPLACEME | 16 + .../jmemcached-core-1.0.0.jar.REPLACEME | 16 + .../memcached-win/package/scripts/memcached.py | 54 ++ .../memcached-win/package/scripts/params.py | 32 + app-packages/memcached-win/resources.json | 16 + app-packages/memcached/README.txt | 35 ++ app-packages/memcached/appConfig.json | 22 + app-packages/memcached/metainfo.xml | 51 ++ .../package/files/jmemcached-1.0.0.tar.REPLACE | 20 + .../memcached/package/scripts/memcached.py | 53 ++ .../memcached/package/scripts/params.py | 32 + app-packages/memcached/resources.json | 16 + app-packages/storm-v0_91/README.txt | 33 -- app-packages/storm-v0_91/appConfig.json | 126 .../storm-v0_91/configuration/global.xml| 39 -- .../storm-v0_91/configuration/storm-site.xml| 587 --- app-packages/storm-v0_91/ganglia_metrics.json | 31 - app-packages/storm-v0_91/jmx_metrics.json | 31 - app-packages/storm-v0_91/metainfo.xml | 145 - ...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 - .../storm-v0_91/package/scripts/drpc_server.py | 55 -- .../storm-v0_91/package/scripts/nimbus.py | 55 -- .../storm-v0_91/package/scripts/params.py | 58 -- .../storm-v0_91/package/scripts/rest_api.py | 57 -- .../storm-v0_91/package/scripts/service.py | 95 --- .../package/scripts/status_params.py| 36 -- .../storm-v0_91/package/scripts/storm.py| 50 -- .../storm-v0_91/package/scripts/supervisor.py | 61 -- .../storm-v0_91/package/scripts/ui_server.py| 55 -- .../storm-v0_91/package/scripts/yaml_config.py | 69 --- .../package/templates/config.yaml.j2| 48 -- .../package/templates/storm_jaas.conf.j2| 26 - app-packages/storm-v0_91/resources.json | 31 - app-packages/storm/README.txt | 37 ++ app-packages/storm/appConfig.json | 126 app-packages/storm/configuration/global.xml | 39 ++ app-packages/storm/configuration/storm-site.xml | 587 +++ app-packages/storm/ganglia_metrics.json | 31 + app-packages/storm/jmx_metrics.json | 31 + app-packages/storm/metainfo.xml | 145 + ...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 + .../storm/package/scripts/drpc_server.py| 55 ++ app-packages/storm/package/scripts/nimbus.py| 55 ++ app-packages/storm/package/scripts/params.py| 58 ++ app-packages/storm/package/scripts/rest_api.py | 57 ++ app-packages/storm/package/scripts/service.py | 95 +++ .../storm/package/scripts/status_params.py | 36 ++ app-packages/storm/package/scripts/storm.py | 50 ++ .../storm/package/scripts/supervisor.py | 61 ++ app-packages/storm/package/scripts/ui_server.py | 55 ++ .../storm/package/scripts/yaml_config.py| 69 +++ .../storm/package/templates/config.yaml.j2 | 48 ++ .../storm/package/templates/s
git commit: SLIDER-149 ProviderServices can register themselves and otherwise manipulate the YARN registry; AM register the liveness znode and itself as an (ephemeral) component
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 7e1c77f89 ->
f7b24156d
SLIDER-149 ProviderServices can register themselves and otherwise manipulate
the YARN registry; AM register the liveness znode and itself as an (ephemeral)
component
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f7b24156
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f7b24156
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f7b24156
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f7b24156d2e146063de0ff7194e89158d301ea6b
Parents: 7e1c77f
Author: Steve Loughran
Authored: Mon Aug 4 15:05:23 2014 +0100
Committer: Steve Loughran
Committed: Mon Aug 4 15:05:23 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 8 +-
.../providers/AbstractProviderService.java | 5 +-
.../slider/providers/ProviderService.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 34 +--
.../server/services/curator/CuratorService.java | 5 +-
.../YarnRegistryViewForProviders.java | 93
.../standalone/TestStandaloneRegistryAM.groovy | 4 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../publisher/TestAgentProviderService.java | 4 +-
.../providers/hbase/HBaseProviderService.java | 19
10 files changed, 157 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7b24156/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 87af625..20caa8c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2341,8 +2341,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @return a class that can dump the contents of the registry
*/
@VisibleForTesting
- public ZKPathDumper dumpSliderRegistry() throws SliderException, IOException
{
-return getRegistry().dumpPath();
+ public ZKPathDumper dumpSliderRegistry(boolean verbose) throws
SliderException, IOException {
+return getRegistry().dumpPath(verbose);
}
/**
@@ -2427,8 +2427,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @return a class that can dump the contents of the registry
*/
@VisibleForTesting
- public ZKPathDumper dumpYarnRegistry() throws SliderException, IOException {
-return getYarnRegistry().dumpPath();
+ public ZKPathDumper dumpYarnRegistry(boolean verbose) throws
SliderException, IOException {
+return getYarnRegistry().dumpPath(verbose);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f7b24156/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index b50b09d..eb8fc73 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -39,6 +39,7 @@ import
org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
+import
org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +71,7 @@ public abstract class AbstractProviderService
protected ServiceInstanceData registryInstanceData;
protected AMViewForProviders amView;
protected URL amWebAPI;
- private YarnRegistryService yarnRegistry;
+ protected YarnRegistryViewForProviders yarnRegistry;
public AbstractProviderService(String name) {
super(name);
@@ -96,7 +97,7 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
RegistryViewForProviders reg,
- YarnRegistryService yarnRegistry,
+ YarnRegistryViewForProviders yarnRegistry,
AMViewForProviders amView) {
this.amState = stateAccessor;
this.registry = reg;
http://git-wip-us.apache.org/repos/a
[2/2] git commit: SLIDER-284 POM bonded to 2.5.0 staging repo
SLIDER-284 POM bonded to 2.5.0 staging repo Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/248b6ddc Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/248b6ddc Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/248b6ddc Branch: refs/heads/feature/SLIDER-284-hadoop2.5 Commit: 248b6ddc368f212dd9ff4b1a580a0722c9e486e7 Parents: 9708410 Author: Steve Loughran Authored: Tue Aug 5 11:14:22 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 5 11:14:22 2014 +0100 -- pom.xml | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/248b6ddc/pom.xml -- diff --git a/pom.xml b/pom.xml index 7cca66c..77931d4 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ -2.4.1 +2.5.0 0.98.4-hadoop2 1.6.0 @@ -203,10 +203,16 @@ + + +Hadoop 2.5.0 RC + https://repository.apache.org/content/repositories/orgapachehadoop-1007/ +
[1/2] git commit: SLIDER-34 SLIDER-284 -re-enabling test that now passes on Hadoop 2.5.0
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-284-hadoop2.5 [created] 248b6ddc3
SLIDER-34 SLIDER-284 -re-enabling test that now passes on Hadoop 2.5.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/97084100
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/97084100
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/97084100
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 97084100f027d969a66c8bcbdbd766839559e3eb
Parents: 7ab9ce9
Author: Steve Loughran
Authored: Tue Aug 5 11:11:36 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 11:11:36 2014 +0100
--
.../hbase/minicluster/failures/TestKilledHBaseAM.groovy | 12
1 file changed, 4 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/97084100/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
index d219636..16a4e62 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
@@ -27,15 +27,16 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
+
import static org.apache.slider.test.SliderTestUtils.log
/**
@@ -47,11 +48,9 @@ class TestKilledHBaseAM extends HBaseMiniClusterTestBase {
@Test
public void testKilledHBaseAM() throws Throwable {
-skip("SLIDER-66: AM Restart Failing -YARN issues")
int regionServerCount = 1
-
def conf = configuration
// patch the configuration for AM restart
conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 3)
@@ -94,8 +93,6 @@ class TestKilledHBaseAM extends HBaseMiniClusterTestBase {
HConnection hbaseConnection
hbaseConnection = createHConnection(clientConf)
-
-
describe("running processes")
lsJavaProcesses()
describe("killing AM")
@@ -115,7 +112,6 @@ class TestKilledHBaseAM extends HBaseMiniClusterTestBase {
ApplicationReport report = sliderClient.applicationReport
assert report.yarnApplicationState != YarnApplicationState.FAILED;
-
def restartTime = 6
status = waitForWorkerInstanceCount(
sliderClient,
git commit: SLIDER-90: HA NN, RM support
Repository: incubator-slider
Updated Branches:
refs/heads/develop 7ab9ce96e -> 0b52d4b0e
SLIDER-90: HA NN, RM support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b52d4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b52d4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b52d4b0
Branch: refs/heads/develop
Commit: 0b52d4b0e2fc0a3a5e98c34bb6248a84e359c3db
Parents: 7ab9ce9
Author: Steve Loughran
Authored: Tue Aug 5 12:04:41 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 12:04:41 2014 +0100
--
.../java/org/apache/slider/client/SliderClient.java | 10 --
.../slider/common/params/AbstractActionArgs.java | 14 +-
.../java/org/apache/slider/common/params/ArgOps.java | 7 ---
.../org/apache/slider/common/params/ClientArgs.java | 5 +++--
.../org/apache/slider/common/params/CommonArgs.java | 11 +--
.../slider/server/appmaster/SliderAppMaster.java | 4 ++--
6 files changed, 23 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..ff2e865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -182,7 +182,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
serviceArgs.applyDefinitions(conf);
-serviceArgs.applyFileSystemURL(conf);
+serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
SliderUtils.forceLogin();
@@ -330,9 +330,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
-} catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
-} catch (KeeperException e) {
+} catch (InterruptedException | KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1008,8 +1006,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
}
-if (serviceArgs.getFilesystemURL() != null) {
- commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemURL());
+if (serviceArgs.getFilesystemBinding() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemBinding());
}
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 44bc239..3a8cea4 100644
---
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -25,7 +25,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -41,26 +40,23 @@ public abstract class AbstractActionArgs extends ArgOps
implements Arguments {
}
/**
- * URI of the filesystem
+ * URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem URI",
- converter = URIArgumentConverter.class)
- public URI filesystemURL;
+ description = "Filesystem Binding")
+ public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
-
/**
* This is the default parameter
*/
@Parameter
public final List parameters = new ArrayList<>();
-
/**
* get the na
[8/9] git commit: SLIDER-270: one-liner fix to actually resolve the problem
SLIDER-270: one-liner fix to actually resolve the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d6c3fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d6c3fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d6c3fad
Branch: refs/heads/develop
Commit: 8d6c3fad0e1edc9c482f43305f82c976e3d53aa9
Parents: df963f2
Author: Steve Loughran
Authored: Wed Aug 6 09:32:36 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 09:33:07 2014 +0100
--
.../apache/slider/server/appmaster/state/AppState.java | 13 ++---
.../server/appmaster/model/mock/MockYarnEngine.groovy | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b17c3a..db74ea8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -786,7 +786,10 @@ public class AppState {
public RoleInstance getActiveContainer(ContainerId id) {
return activeContainers.get(id);
}
-
+
+ private RoleInstance removeActiveContainer(ContainerId id) {
+return activeContainers.remove(id);
+ }
public synchronized List cloneLiveContainerInfoList() {
List allRoleInstances;
@@ -1265,11 +1268,15 @@ public class AppState {
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
+
+// and the active node list if present
+removeActiveContainer(containerId);
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
-containerId) : "container in release queue";
+containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
-containerId) : " container in live nodes";
+containerId) : " container still in live nodes";
assert getActiveContainer(containerId) ==
null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 763f653..b57a79a 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -104,7 +104,7 @@ class MockYarnEngine {
if (op instanceof ContainerReleaseOperation) {
ContainerReleaseOperation cro = (ContainerReleaseOperation) op
ContainerId cid = cro.containerId
-releaseContainer(cid);
+assert releaseContainer(cid);
released.add(cid)
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
[1/9] git commit: SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
Repository: incubator-slider
Updated Branches:
refs/heads/develop 405aa51e2 -> d68be5aa5
SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock
test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19
Branch: refs/heads/develop
Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29
Parents: 0b52d4b
Author: Steve Loughran
Authored: Tue Aug 5 16:05:03 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 16:05:03 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 16 +--
.../slider/server/appmaster/state/AppState.java | 43 +++
.../appstate/TestAppStateDynamicRoles.groovy| 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++
.../model/appstate/TestMockRMOperations.groovy | 37 --
.../model/mock/BaseMockAppStateTest.groovy | 57 -
.../appmaster/model/mock/MockYarnEngine.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
11 files changed, 219 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..1ad7fa7 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -671,12 +671,13 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
//build the instance
appState.buildInstance(instanceDefinition,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation);
+ serviceConf,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
// add the AM to the list of nodes in the cluster
@@ -1055,8 +1056,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// non complete containers should not be here
assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- getConfig(), status);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(status);
if (result.containerFailed) {
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index cc238ff..9f8a39d 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.providers.ProviderRole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,6 +249,8 @@ public class AppState {
private long startTimeThreshold;
private int failureThreshold = 10;
+
+ private String logServerURL = "";
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -432,6 +433,7 @@ public class AppState {
/**
* Build up the application state
* @param instanceDefinition definition of the applicatin instance
+ * @param appmasterConfig
* @param publishedProviderConf any configuration info to be
[7/9] git commit: SLIDER-270 assertion checking has found the location of the problem
SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24
Branch: refs/heads/develop
Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f
Parents: 291a82e
Author: Steve Loughran
Authored: Tue Aug 5 21:35:28 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:35:54 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 25 ++--
.../server/appmaster/state/RoleHistory.java | 6 ++---
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../src/test/resources/log4j.properties | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 74e91a0..2b17c3a 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1172,7 +1172,7 @@ public class AppState {
RoleInstance roleInstance;
if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release");
+ log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
int releasing = roleStatus.decReleasing();
@@ -1183,7 +1183,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container);
+ roleHistory.onReleaseCompleted(container, true);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1251,18 +1251,28 @@ public class AppState {
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
+log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
-if (node == null) {
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
-
-} else {
+if (node != null) {
node.state = ClusterDescription.STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
result.roleInstance = node;
+} else {
+ // not in the list
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
}
+// finally, verify the node doesn't exist any more
+assert !containersBeingReleased.containsKey(
+containerId) : "container in release queue";
+assert !getLiveNodes().containsKey(
+containerId) : " container in live nodes";
+assert getActiveContainer(containerId) ==
+ null : "Container still in active container list";
+
return result;
}
@@ -1422,6 +1432,7 @@ public class AppState {
* @throws SliderInternalStateException if the operation reveals that
* the internal state of the application is inconsistent.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public List reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List operations = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0cd2b39..de7e9d8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.R
[6/9] git commit: SLIDER-270 more work on the double flex test case
SLIDER-270 more work on the double flex test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/291a82ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/291a82ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/291a82ef
Branch: refs/heads/develop
Commit: 291a82ef1e99a6590e245ebce7c0d1d861eac263
Parents: 040ef79
Author: Steve Loughran
Authored: Tue Aug 5 21:09:00 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:09:00 2014 +0100
--
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ---
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 1 +
.../minicluster/flexing/TestClusterFlexDownMultiple.groovy| 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 49dc02e..ed66190 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -136,6 +136,9 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
}
}
+ protected String buildClustername(String clustername) {
+return clustername ?: createClusterName()
+ }
/**
* Create the cluster name from the method name and an auto-incrementing
@@ -240,9 +243,7 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
-if (!name) {
- name = createClusterName()
-}
+name = buildClustername(name)
miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs,
numLogDirs)
miniCluster.init(conf)
miniCluster.start();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index c9f0a4f..6a69e17 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -284,6 +284,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int workers,
int workerFlexTarget,
boolean testHBaseAfter) {
+clustername = buildClustername(clustername);
SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
//now flex
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
index 43ade73..97a9b35 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -34,13 +34,13 @@ class TestClusterFlexDownMultiple extends
HBaseMiniClusterTestBase {
@Test
public void testClusterFlexDownMultiple() throws Throwable {
def clusterName = createClusterName();
-SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+SliderClient sliderClient = startHBaseCluster(clusterName, 1, 3)
assert flexCluste
[9/9] git commit: Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop
Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d68be5aa Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d68be5aa Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d68be5aa Branch: refs/heads/develop Commit: d68be5aa5eaa8c2e5a2d71097eb2907968c512bd Parents: 405aa51 8d6c3fa Author: Steve Loughran Authored: Wed Aug 6 09:33:34 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 09:33:34 2014 +0100 -- pom.xml | 2 +- .../server/appmaster/SliderAppMaster.java | 16 +- .../slider/server/appmaster/state/AppState.java | 142 +++--- .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 4 +- .../appstate/TestAppStateDynamicRoles.groovy| 4 +- .../TestAppStateRebuildOnAMRestart.groovy | 4 +- .../model/appstate/TestFlexDynamicRoles.groovy | 2 + .../model/appstate/TestMockFlexing.groovy | 128 .../model/appstate/TestMockRMOperations.groovy | 37 - .../TestRoleHistoryContainerEvents.groovy | 2 +- .../model/mock/BaseMockAppStateTest.groovy | 57 ++- .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 1 + .../minicluster/HBaseMiniClusterTestBase.groovy | 150 --- .../flexing/TestClusterFlex2To5.groovy | 2 +- .../flexing/TestClusterFlexDownMultiple.groovy | 58 +++ .../TestClusterFlexHMasterFlex1To2.groovy | 45 ++ .../flexing/TestHMasterFlex1To2.groovy | 45 -- .../src/test/resources/log4j.properties | 1 + 22 files changed, 497 insertions(+), 225 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d68be5aa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[3/9] git commit: SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex tests
SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex
tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4d0734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4d0734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4d0734
Branch: refs/heads/develop
Commit: ae4d07340b0c0fe54b3dc75e58744393d5949d31
Parents: 34c00e8
Author: Steve Loughran
Authored: Tue Aug 5 21:06:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:06:54 2014 +0100
--
.../TestClusterFlexHMasterFlex1To2.groovy | 45
.../flexing/TestHMasterFlex1To2.groovy | 45
2 files changed, 45 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
new file mode 100644
index 000..298f40f
--- /dev/null
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterMasterFlex1To2() throws Throwable {
+assert flexHBaseClusterTestRun(
+"",
+1,
+2,
+1,
+1,
+true)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
deleted file mode 100644
index ca26572..000
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.minicluster.flexing
-
-import groovy.transform.CompileStatic
-i
[2/9] git commit: SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/develop
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 18:10:59 2014 +0100
--
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ---
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58
3 files changed, 151 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
-extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
[:])
}
+ public List hbaseArgs(List extraArgs) {
+extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
-[
-ARG_PROVIDER, PROVIDER_HBASE
-],
+hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
-createMiniCluster(clustername, configuration,
- 1,
- true);
+SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
+
+//now flex
+return flexCluster(
+sliderClient,
+clustername,
+masterFlexTarget,
+workerFlexTarget,
+testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+clustername = createMiniCluster(clustername, configuration,
+1,
+true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true,
- [:]);
+[
+
[5/9] git commit: SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT
SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/040ef798 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/040ef798 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/040ef798 Branch: refs/heads/develop Commit: 040ef79802d0f9c3a77a4b64a5fbea0d05f67fb0 Parents: 268253c Author: Steve Loughran Authored: Tue Aug 5 21:08:38 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 5 21:08:38 2014 +0100 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/040ef798/pom.xml -- diff --git a/pom.xml b/pom.xml index 7cca66c..253d40f 100644 --- a/pom.xml +++ b/pom.xml @@ -1264,7 +1264,7 @@ branch-2 -2.5.0-SNAPSHOT +2.6.0-SNAPSHOT
[4/9] git commit: SLIDER-270 improving diags and failure log generation in AppState release operations
SLIDER-270 improving diags and failure log generation in AppState release
operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/268253c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/268253c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/268253c1
Branch: refs/heads/develop
Commit: 268253c1bb1582d4d1879c4798064e59bb447da0
Parents: ae4d073
Author: Steve Loughran
Authored: Tue Aug 5 21:07:23 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:08:05 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 69 +---
.../server/appmaster/state/RoleStatus.java | 4 +-
2 files changed, 46 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f8a39d..74e91a0 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -696,7 +696,7 @@ public class AppState {
roleStatusMap.get(priority));
}
roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+new RoleStatus(providerRole));
roles.put(providerRole.name, providerRole);
}
@@ -1175,10 +1175,14 @@ public class AppState {
log.info("Container was queued for release");
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- log.info("decrementing role count for role {}", roleStatus.getName());
- roleStatus.decReleasing();
- roleStatus.decActual();
- roleStatus.incCompleted();
+ int releasing = roleStatus.decReleasing();
+ int actual = roleStatus.decActual();
+ int completedCount = roleStatus.incCompleted();
+ log.info("decrementing role count for role {} to {}; releasing={},
completed={}",
+ roleStatus.getName(),
+ actual,
+ releasing,
+ completedCount);
roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
@@ -1205,25 +1209,17 @@ public class AppState {
roleStatus.decActual();
boolean shortLived = isShortLived(roleInstance);
String message;
- if (roleInstance.container != null) {
-String user = null;
-try {
- user = SliderUtils.getCurrentUser().getShortUserName();
-} catch (IOException ignored) {
-}
-String completedLogsUrl = null;
-Container c = roleInstance.container;
-String url = logServerURL;
-if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + roleInstance.getContainerId()
+ "/ctx/" + user;
-}
-message = String.format("Failure %s on host %s" +
-(completedLogsUrl != null ? ", see %s" : ""),
roleInstance.getContainerId(),
-c.getNodeId().getHost(), completedLogsUrl);
+ Container failedContainer = roleInstance.container;
+
+ //build the failure message
+ if (failedContainer != null) {
+String completedLogsUrl = getLogsURLForContainer(failedContainer);
+message = String.format("Failure %s on host %s: %s" +
+roleInstance.getContainerId(),
+failedContainer.getNodeId().getHost(),
+completedLogsUrl);
} else {
-message = String.format("Failure %s",
-containerId.toString());
+message = String.format("Failure %s", containerId);
}
roleStatus.noteFailed(message);
//have a look to see if it short lived
@@ -1231,8 +1227,8 @@ public class AppState {
roleStatus.incStartFailed();
}
- if (roleInstance.container != null) {
-roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ if (failedContainer!= null) {
+roleHistory.onFailedContainer(failedContainer, shortLived);
}
} catch (YarnRuntimeException e1) {
@@ -1270,6 +1266,29 @@ public class AppState {
return result;
}
+ /**
+ * Get the URL log for a container
+ * @param c container
+ *
[09/18] git commit: SLIDER-270 assertion checking has found the location of the problem
SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f
Parents: 291a82e
Author: Steve Loughran
Authored: Tue Aug 5 21:35:28 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:35:54 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 25 ++--
.../server/appmaster/state/RoleHistory.java | 6 ++---
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../src/test/resources/log4j.properties | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 74e91a0..2b17c3a 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1172,7 +1172,7 @@ public class AppState {
RoleInstance roleInstance;
if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release");
+ log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
int releasing = roleStatus.decReleasing();
@@ -1183,7 +1183,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container);
+ roleHistory.onReleaseCompleted(container, true);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1251,18 +1251,28 @@ public class AppState {
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
+log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
-if (node == null) {
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
-
-} else {
+if (node != null) {
node.state = ClusterDescription.STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
result.roleInstance = node;
+} else {
+ // not in the list
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
}
+// finally, verify the node doesn't exist any more
+assert !containersBeingReleased.containsKey(
+containerId) : "container in release queue";
+assert !getLiveNodes().containsKey(
+containerId) : " container in live nodes";
+assert getActiveContainer(containerId) ==
+ null : "Container still in active container list";
+
return result;
}
@@ -1422,6 +1432,7 @@ public class AppState {
* @throws SliderInternalStateException if the operation reveals that
* the internal state of the application is inconsistent.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public List reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List operations = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0cd2b39..de7e9d8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache
[05/18] git commit: SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex tests
SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex
tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4d0734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4d0734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4d0734
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: ae4d07340b0c0fe54b3dc75e58744393d5949d31
Parents: 34c00e8
Author: Steve Loughran
Authored: Tue Aug 5 21:06:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:06:54 2014 +0100
--
.../TestClusterFlexHMasterFlex1To2.groovy | 45
.../flexing/TestHMasterFlex1To2.groovy | 45
2 files changed, 45 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
new file mode 100644
index 000..298f40f
--- /dev/null
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterMasterFlex1To2() throws Throwable {
+assert flexHBaseClusterTestRun(
+"",
+1,
+2,
+1,
+1,
+true)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
deleted file mode 100644
index ca26572..000
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.minicluster.flexing
-
-import groovy.trans
[18/18] git commit: Merge branch 'develop' into feature/SLIDER-284-hadoop2.5
Merge branch 'develop' into feature/SLIDER-284-hadoop2.5 Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ef22cb29 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ef22cb29 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ef22cb29 Branch: refs/heads/feature/SLIDER-284-hadoop2.5 Commit: ef22cb2965a2216b98a8bdfa2db98cceb2598ee2 Parents: 248b6dd d68be5a Author: Steve Loughran Authored: Wed Aug 6 09:35:10 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 09:35:10 2014 +0100 -- app-packages/hbase/appConfig.json | 2 +- app-packages/hbase/metainfo.xml | 15 + .../hbase/package/scripts/hbase_service.py | 2 + .../hbase/package/scripts/hbase_thrift.py | 62 +++ app-packages/hbase/package/scripts/params.py| 1 + app-packages/hbase/resources.json | 5 + app-packages/memcached-win/appConfig.json | 6 +- app-packages/memcached-win/metainfo.xml | 8 +- .../memcached-win/package/scripts/memcached.py | 2 +- .../memcached-win/package/scripts/params.py | 1 + app-packages/memcached/appConfig.json | 7 +- app-packages/memcached/metainfo.xml | 8 +- .../memcached/package/scripts/memcached.py | 2 +- .../memcached/package/scripts/params.py | 1 + pom.xml | 2 +- .../python/agent/CustomServiceOrchestrator.py | 86 +++- .../core/providers/windows/system.py| 54 +- .../agent/TestCustomServiceOrchestrator.py | 127 - slider-agent/src/test/python/agent/TestGrep.py | 2 +- slider-agent/src/test/python/agent/TestMain.py | 8 +- .../src/test/python/agent/TestRegistration.py | 4 +- .../org/apache/slider/client/SliderClient.java | 10 +- .../common/params/AbstractActionArgs.java | 14 +- .../org/apache/slider/common/params/ArgOps.java | 7 +- .../apache/slider/common/params/ClientArgs.java | 5 +- .../apache/slider/common/params/CommonArgs.java | 11 +- .../providers/AbstractProviderService.java | 5 + .../slider/providers/ProviderService.java | 7 + .../providers/agent/AgentProviderService.java | 510 --- .../agent/application/metadata/Application.java | 9 + .../agent/application/metadata/Component.java | 13 + .../application/metadata/MetainfoParser.java| 5 + .../server/appmaster/SliderAppMaster.java | 22 +- .../slider/server/appmaster/state/AppState.java | 142 +++--- .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 4 +- .../appstate/TestAppStateDynamicRoles.groovy| 4 +- .../TestAppStateRebuildOnAMRestart.groovy | 4 +- .../model/appstate/TestFlexDynamicRoles.groovy | 2 + .../model/appstate/TestMockFlexing.groovy | 128 + .../model/appstate/TestMockRMOperations.groovy | 37 -- .../TestRoleHistoryContainerEvents.groovy | 2 +- .../model/mock/BaseMockAppStateTest.groovy | 57 ++- .../model/mock/MockProviderService.groovy | 5 + .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../agent/TestAgentProviderService.java | 137 - .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 1 + .../publisher/TestAgentProviderService.java | 4 +- .../providers/hbase/HBaseClientProvider.java| 2 +- .../slider/providers/hbase/HBaseKeys.java | 5 + .../providers/hbase/HBaseProviderService.java | 5 + .../slider/providers/hbase/HBaseRoles.java | 3 + .../minicluster/HBaseMiniClusterTestBase.groovy | 150 +++--- .../flexing/TestClusterFlex2To5.groovy | 2 +- .../flexing/TestClusterFlexDownMultiple.groovy | 58 +++ .../TestClusterFlexHMasterFlex1To2.groovy | 45 ++ .../flexing/TestHMasterFlex1To2.groovy | 45 -- .../src/test/resources/log4j.properties | 1 + 60 files changed, 1392 insertions(+), 496 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ef22cb29/pom.xml --
[06/18] git commit: SLIDER-270 improving diags and failure log generation in AppState release operations
SLIDER-270 improving diags and failure log generation in AppState release
operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/268253c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/268253c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/268253c1
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 268253c1bb1582d4d1879c4798064e59bb447da0
Parents: ae4d073
Author: Steve Loughran
Authored: Tue Aug 5 21:07:23 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:08:05 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 69 +---
.../server/appmaster/state/RoleStatus.java | 4 +-
2 files changed, 46 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f8a39d..74e91a0 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -696,7 +696,7 @@ public class AppState {
roleStatusMap.get(priority));
}
roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+new RoleStatus(providerRole));
roles.put(providerRole.name, providerRole);
}
@@ -1175,10 +1175,14 @@ public class AppState {
log.info("Container was queued for release");
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- log.info("decrementing role count for role {}", roleStatus.getName());
- roleStatus.decReleasing();
- roleStatus.decActual();
- roleStatus.incCompleted();
+ int releasing = roleStatus.decReleasing();
+ int actual = roleStatus.decActual();
+ int completedCount = roleStatus.incCompleted();
+ log.info("decrementing role count for role {} to {}; releasing={},
completed={}",
+ roleStatus.getName(),
+ actual,
+ releasing,
+ completedCount);
roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
@@ -1205,25 +1209,17 @@ public class AppState {
roleStatus.decActual();
boolean shortLived = isShortLived(roleInstance);
String message;
- if (roleInstance.container != null) {
-String user = null;
-try {
- user = SliderUtils.getCurrentUser().getShortUserName();
-} catch (IOException ignored) {
-}
-String completedLogsUrl = null;
-Container c = roleInstance.container;
-String url = logServerURL;
-if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + roleInstance.getContainerId()
+ "/ctx/" + user;
-}
-message = String.format("Failure %s on host %s" +
-(completedLogsUrl != null ? ", see %s" : ""),
roleInstance.getContainerId(),
-c.getNodeId().getHost(), completedLogsUrl);
+ Container failedContainer = roleInstance.container;
+
+ //build the failure message
+ if (failedContainer != null) {
+String completedLogsUrl = getLogsURLForContainer(failedContainer);
+message = String.format("Failure %s on host %s: %s" +
+roleInstance.getContainerId(),
+failedContainer.getNodeId().getHost(),
+completedLogsUrl);
} else {
-message = String.format("Failure %s",
-containerId.toString());
+message = String.format("Failure %s", containerId);
}
roleStatus.noteFailed(message);
//have a look to see if it short lived
@@ -1231,8 +1227,8 @@ public class AppState {
roleStatus.incStartFailed();
}
- if (roleInstance.container != null) {
-roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ if (failedContainer!= null) {
+roleHistory.onFailedContainer(failedContainer, shortLived);
}
} catch (YarnRuntimeException e1) {
@@ -1270,6 +1266,29 @@ public class AppState {
return result;
}
+ /**
+ * Get the URL log for a container
+ * @pa
[15/18] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1894a0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1894a0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1894a0ca
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 1894a0ca206d4d024f7b3062133ab1039ca5988b
Parents: a9a91b4
Author: Sumit Mohanty
Authored: Tue Jul 29 23:33:00 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached/appConfig.json | 3 +-
app-packages/memcached/metainfo.xml | 6 ++
.../python/agent/CustomServiceOrchestrator.py | 57 --
.../agent/TestCustomServiceOrchestrator.py | 107 +--
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../src/test/python/agent/TestRegistration.py | 4 +-
.../providers/agent/AgentProviderService.java | 3 +
.../agent/application/metadata/Component.java | 13 +++
.../application/metadata/MetainfoParser.java| 4 +
.../agent/TestAgentProviderService.java | 25 -
11 files changed, 202 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index d205591..11fb863 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -10,7 +10,8 @@
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
-"site.global.memory_val": "200M"
+"site.global.memory_val": "200M",
+"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/metainfo.xml
--
diff --git a/app-packages/memcached/metainfo.xml
b/app-packages/memcached/metainfo.xml
index d435b84..0550b7d 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -27,6 +27,12 @@
MEMCACHED
MASTER
+
+
+host_port
+${THIS_HOST}:${site.global.listen_port}
+
+
scripts/memcached.py
PYTHON
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 4eb22bf..7d740c2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -52,9 +52,9 @@ class CustomServiceOrchestrator():
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stdout.txt'))
+
'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stderr.txt'))
+
'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
@@ -81,7 +81,7 @@ class CustomServiceOrchestrator():
script_tuple = (script_path, self.base_dir)
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
-
"structured-out-{0}.json".format(task_id)))
+
"structured-out-{0}.json".format(task_id)))
if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
@@ -97,9 +97,9 @@ class CustomServiceOrchestrator():
for py_file, current_base_dir in filtered_py_file_list:
script_params = [command_name, json_path, current_base_dir]
python_paths =
[os.path.realpath(posixpath.join(self.config.get
[11/18] git commit: SLIDER-273. Add ability to specify a port as shared across component instances or not
SLIDER-273. Add ability to specify a port as shared across component instances
or not
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/50dc76c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/50dc76c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/50dc76c9
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 50dc76c978ef4ff684263b93e8b56619cc947284
Parents: dca4818
Author: Sumit Mohanty
Authored: Sun Aug 3 19:30:13 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached/appConfig.json | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 27 +++-
.../agent/TestCustomServiceOrchestrator.py | 18 ++-
.../providers/AbstractProviderService.java | 5 +
.../providers/agent/AgentProviderService.java | 126 ---
.../model/mock/MockProviderService.groovy | 5 +
.../agent/TestAgentProviderService.java | 58 +++--
.../publisher/TestAgentProviderService.java | 4 +-
8 files changed, 185 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index 11fb863..78cb486 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -11,7 +11,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
-"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
+"site.global.listen_port":
"${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index f6cdfb2..0f4acc6 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -233,12 +233,17 @@ class CustomServiceOrchestrator():
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
+ ALLOCATED_PORT is a hint to allocate port. It works as follows:
+ Its of the form
{component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Either a port gets allocated or if not then just set the value to "0"
"""
def finalize_command(self, command, store_config, allocated_ports):
component = command['componentName']
-allocated_port_format = "${{{0}.ALLOCATED_PORT}}"
-port_allocation_req = allocated_port_format.format(component)
+allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
+allocated_for_any = ".ALLOCATED_PORT}"
+
+port_allocation_req = allocated_for_this_component_format.format(component)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -251,6 +256,10 @@ class CustomServiceOrchestrator():
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
allocated_ports[key + "." + k] = value
+ elif allocated_for_any in value:
+## All unallocated ports should be set to 0
+logger.info("Assigning port 0 " + "for " + value)
+value = "0"
command['configurations'][key][k] = value
pass
pass
@@ -264,8 +273,15 @@ class CustomServiceOrchestrator():
pass
+ """
+ Port allocation can asks for multiple dynamic ports
+ port_req_pattern is of type ${component_name.ALLOCATED_PORT}
+append {DEFAULT_ and find the default value
+append {DO_NOT_PROPAGATE} if it exists
+ """
def allocate_ports(self, value, port_req_pattern):
default_port_pattern = "{DEFAULT_"
+do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
@@ -276,8 +292,15 @@ class CustomServiceOrchestrator():
end_index = value.find("}", start_index)
def_port_str = value[start_index:end_index]
def_port = int(def_port_str)
+# default value of 0 means allocate any dynamic port
+if def_port == 0:
+ def_port = None
+
replaced_pattern
[17/18] git commit: Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop
Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d68be5aa Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d68be5aa Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d68be5aa Branch: refs/heads/feature/SLIDER-284-hadoop2.5 Commit: d68be5aa5eaa8c2e5a2d71097eb2907968c512bd Parents: 405aa51 8d6c3fa Author: Steve Loughran Authored: Wed Aug 6 09:33:34 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 09:33:34 2014 +0100 -- pom.xml | 2 +- .../server/appmaster/SliderAppMaster.java | 16 +- .../slider/server/appmaster/state/AppState.java | 142 +++--- .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 4 +- .../appstate/TestAppStateDynamicRoles.groovy| 4 +- .../TestAppStateRebuildOnAMRestart.groovy | 4 +- .../model/appstate/TestFlexDynamicRoles.groovy | 2 + .../model/appstate/TestMockFlexing.groovy | 128 .../model/appstate/TestMockRMOperations.groovy | 37 - .../TestRoleHistoryContainerEvents.groovy | 2 +- .../model/mock/BaseMockAppStateTest.groovy | 57 ++- .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 1 + .../minicluster/HBaseMiniClusterTestBase.groovy | 150 --- .../flexing/TestClusterFlex2To5.groovy | 2 +- .../flexing/TestClusterFlexDownMultiple.groovy | 58 +++ .../TestClusterFlexHMasterFlex1To2.groovy | 45 ++ .../flexing/TestHMasterFlex1To2.groovy | 45 -- .../src/test/resources/log4j.properties | 1 + 22 files changed, 497 insertions(+), 225 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d68be5aa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[08/18] git commit: SLIDER-270 more work on the double flex test case
SLIDER-270 more work on the double flex test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/291a82ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/291a82ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/291a82ef
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 291a82ef1e99a6590e245ebce7c0d1d861eac263
Parents: 040ef79
Author: Steve Loughran
Authored: Tue Aug 5 21:09:00 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:09:00 2014 +0100
--
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ---
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 1 +
.../minicluster/flexing/TestClusterFlexDownMultiple.groovy| 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 49dc02e..ed66190 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -136,6 +136,9 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
}
}
+ protected String buildClustername(String clustername) {
+return clustername ?: createClusterName()
+ }
/**
* Create the cluster name from the method name and an auto-incrementing
@@ -240,9 +243,7 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
-if (!name) {
- name = createClusterName()
-}
+name = buildClustername(name)
miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs,
numLogDirs)
miniCluster.init(conf)
miniCluster.start();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index c9f0a4f..6a69e17 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -284,6 +284,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int workers,
int workerFlexTarget,
boolean testHBaseAfter) {
+clustername = buildClustername(clustername);
SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
//now flex
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
index 43ade73..97a9b35 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -34,13 +34,13 @@ class TestClusterFlexDownMultiple extends
HBaseMiniClusterTestBase {
@Test
public void testClusterFlexDownMultiple() throws Throwable {
def clusterName = createClusterName();
-SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+SliderClient sliderClient = startHBaseCluster(clusterName, 1, 3)
[03/18] git commit: SLIDER-283 Add role for HBase THRIFT gateway
SLIDER-283 Add role for HBase THRIFT gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9a91b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9a91b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9a91b46
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: a9a91b46662ecf969cbc7c5e69cef214fc91e086
Parents: 0b52d4b
Author: tedyu
Authored: Tue Aug 5 09:32:53 2014 -0700
Committer: tedyu
Committed: Tue Aug 5 09:32:53 2014 -0700
--
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/metainfo.xml | 15 +
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift.py | 62
app-packages/hbase/package/scripts/params.py| 1 +
app-packages/hbase/resources.json | 5 ++
.../providers/hbase/HBaseClientProvider.java| 2 +-
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 3 +
10 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 8b0825d..f0ab144 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -23,7 +23,7 @@
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
-"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/metainfo.xml
--
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index c14b06f..ee6ddea 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}
+org.apache.slider.hbase.thrift
+
http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}
+
+
org.apache.slider.metrics
http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}
@@ -63,6 +67,7 @@
HBASE_MASTER-START
HBASE_REST-INSTALLED
+HBASE_THRIFT-INSTALLED
@@ -99,6 +104,16 @@
+HBASE_THRIFT
+SLAVE
+1
+
+ scripts/hbase_thrift.py
+ PYTHON
+
+
+
+
HBASE_CLIENT
CLIENT
0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_service.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_service.py
b/app-packages/hbase/package/scripts/hbase_service.py
index cb9c599..01d38b3 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -37,6 +37,8 @@ def hbase_service(
daemon_cmd = format("{cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
+ elif name == 'thrift':
+daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat
{pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_thrift.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_thrift.py
b/app-packages/hbase/package/scripts/hbase_thrift.py
new file mode 100644
index 000..84bfc62
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this f
[10/18] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dca4818c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dca4818c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dca4818c
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: dca4818c551c8fff655c8898c0786aae2fb0fb6a
Parents: ad2c798
Author: Sumit Mohanty
Authored: Fri Aug 1 11:52:35 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
.../slider/providers/ProviderService.java | 7 +
.../providers/agent/AgentProviderService.java | 329 ++-
.../server/appmaster/SliderAppMaster.java | 2 +
.../agent/TestAgentProviderService.java | 49 ++-
4 files changed, 224 insertions(+), 163 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 56e24e9..b9fa34c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -69,6 +70,12 @@ public interface ProviderService extends ProviderCore,
Service,
SliderException;
/**
+ * Notify the providers of container completion
+ * @param containerId
+ */
+ void notifyContainerCompleted(ContainerId containerId);
+
+ /**
* Execute a process in the AM
* @param instanceDefinition cluster description
* @param confDir configuration directory
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b39601a..bbe2001 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
@@ -110,6 +111,7 @@ public class AgentProviderService extends
AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
+ private static final String SHARED_PORT_TAG = "{SHARED}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -290,116 +292,6 @@ public class AgentProviderService extends
AbstractProviderService implements
}
/**
- * Reads and sets the heartbeat monitoring interval. If bad value is
provided then log it and set to default.
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf
instanceDefinition) {
-String hbMonitorInterval = instanceDefinition.getAppConfOperations().
-getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
-
Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
-try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
-}catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
-}
- }
-
- /**
- *
[04/18] git commit: SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 18:10:59 2014 +0100
--
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ---
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58
3 files changed, 151 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
-extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
[:])
}
+ public List hbaseArgs(List extraArgs) {
+extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
-[
-ARG_PROVIDER, PROVIDER_HBASE
-],
+hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
-createMiniCluster(clustername, configuration,
- 1,
- true);
+SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
+
+//now flex
+return flexCluster(
+sliderClient,
+clustername,
+masterFlexTarget,
+workerFlexTarget,
+testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+clustername = createMiniCluster(clustername, configuration,
+1,
+true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true,
- [:]);
+
[01/18] git commit: SLIDER-90: HA NN, RM support
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-284-hadoop2.5 248b6ddc3 -> ef22cb296
SLIDER-90: HA NN, RM support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b52d4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b52d4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b52d4b0
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 0b52d4b0e2fc0a3a5e98c34bb6248a84e359c3db
Parents: 7ab9ce9
Author: Steve Loughran
Authored: Tue Aug 5 12:04:41 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 12:04:41 2014 +0100
--
.../java/org/apache/slider/client/SliderClient.java | 10 --
.../slider/common/params/AbstractActionArgs.java | 14 +-
.../java/org/apache/slider/common/params/ArgOps.java | 7 ---
.../org/apache/slider/common/params/ClientArgs.java | 5 +++--
.../org/apache/slider/common/params/CommonArgs.java | 11 +--
.../slider/server/appmaster/SliderAppMaster.java | 4 ++--
6 files changed, 23 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..ff2e865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -182,7 +182,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
serviceArgs.applyDefinitions(conf);
-serviceArgs.applyFileSystemURL(conf);
+serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
SliderUtils.forceLogin();
@@ -330,9 +330,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
-} catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
-} catch (KeeperException e) {
+} catch (InterruptedException | KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1008,8 +1006,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
}
-if (serviceArgs.getFilesystemURL() != null) {
- commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemURL());
+if (serviceArgs.getFilesystemBinding() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemBinding());
}
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 44bc239..3a8cea4 100644
---
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -25,7 +25,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -41,26 +40,23 @@ public abstract class AbstractActionArgs extends ArgOps
implements Arguments {
}
/**
- * URI of the filesystem
+ * URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem URI",
- converter = URIArgumentConverter.class)
- public URI filesystemURL;
+ description = "Filesystem Binding")
+ public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
-
/**
* This is the default parameter
*/
@Parameter
public final List parameters = new
[16/18] git commit: SLIDER-270: one-liner fix to actually resolve the problem
SLIDER-270: one-liner fix to actually resolve the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d6c3fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d6c3fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d6c3fad
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 8d6c3fad0e1edc9c482f43305f82c976e3d53aa9
Parents: df963f2
Author: Steve Loughran
Authored: Wed Aug 6 09:32:36 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 09:33:07 2014 +0100
--
.../apache/slider/server/appmaster/state/AppState.java | 13 ++---
.../server/appmaster/model/mock/MockYarnEngine.groovy | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b17c3a..db74ea8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -786,7 +786,10 @@ public class AppState {
public RoleInstance getActiveContainer(ContainerId id) {
return activeContainers.get(id);
}
-
+
+ private RoleInstance removeActiveContainer(ContainerId id) {
+return activeContainers.remove(id);
+ }
public synchronized List cloneLiveContainerInfoList() {
List allRoleInstances;
@@ -1265,11 +1268,15 @@ public class AppState {
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
+
+// and the active node list if present
+removeActiveContainer(containerId);
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
-containerId) : "container in release queue";
+containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
-containerId) : " container in live nodes";
+containerId) : " container still in live nodes";
assert getActiveContainer(containerId) ==
null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 763f653..b57a79a 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -104,7 +104,7 @@ class MockYarnEngine {
if (op instanceof ContainerReleaseOperation) {
ContainerReleaseOperation cro = (ContainerReleaseOperation) op
ContainerId cid = cro.containerId
-releaseContainer(cid);
+assert releaseContainer(cid);
released.add(cid)
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
[02/18] git commit: SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock
test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29
Parents: 0b52d4b
Author: Steve Loughran
Authored: Tue Aug 5 16:05:03 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 16:05:03 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 16 +--
.../slider/server/appmaster/state/AppState.java | 43 +++
.../appstate/TestAppStateDynamicRoles.groovy| 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++
.../model/appstate/TestMockRMOperations.groovy | 37 --
.../model/mock/BaseMockAppStateTest.groovy | 57 -
.../appmaster/model/mock/MockYarnEngine.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
11 files changed, 219 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..1ad7fa7 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -671,12 +671,13 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
//build the instance
appState.buildInstance(instanceDefinition,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation);
+ serviceConf,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
// add the AM to the list of nodes in the cluster
@@ -1055,8 +1056,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// non complete containers should not be here
assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- getConfig(), status);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(status);
if (result.containerFailed) {
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index cc238ff..9f8a39d 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.providers.ProviderRole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,6 +249,8 @@ public class AppState {
private long startTimeThreshold;
private int failureThreshold = 10;
+
+ private String logServerURL = "";
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -432,6 +433,7 @@ public class AppState {
/**
* Build up the application state
* @param instanceDefinition definition of the applicatin instance
+ * @param appmasterConfig
* @param publishedProviderConf any configuration info to be published by a
provider
* @param providerRoles roles offered by a p
[14/18] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad2c798d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad2c798d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad2c798d
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: ad2c798d309aded2ef18dea7aa1f90acdd367b14
Parents: 1894a0c
Author: Sumit Mohanty
Authored: Wed Jul 30 23:44:22 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
.../memcached/package/scripts/memcached.py | 2 +-
.../memcached/package/scripts/params.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 2 +-
.../agent/TestCustomServiceOrchestrator.py | 6 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../providers/agent/AgentProviderService.java | 70 ++--
.../agent/TestAgentProviderService.java | 4 +-
7 files changed, 76 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/memcached.py
--
diff --git a/app-packages/memcached/package/scripts/memcached.py
b/app-packages/memcached/package/scripts/memcached.py
index b2df89a..cf04cf1 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -33,7 +33,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
-process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val}")
+process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/params.py
--
diff --git a/app-packages/memcached/package/scripts/params.py
b/app-packages/memcached/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp =
config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 7d740c2..f6cdfb2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -250,7 +250,7 @@ class CustomServiceOrchestrator():
self.config.getLogPath())
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
-allocated_ports[k] = value
+allocated_ports[key + "." + k] = value
command['configurations'][key][k] = value
pass
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
--
diff --git
a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3b41bd7..3343a8b 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -226,7 +226,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 0)
-self.assertEqual(ret['allocated_ports'], {'a.port': '10233'})
+self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 1)
@@ -480,8 +480,8 @@ class TestCustomServiceOrchestrator(TestCase):
self.ass
[07/18] git commit: SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT
SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/040ef798 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/040ef798 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/040ef798 Branch: refs/heads/feature/SLIDER-284-hadoop2.5 Commit: 040ef79802d0f9c3a77a4b64a5fbea0d05f67fb0 Parents: 268253c Author: Steve Loughran Authored: Tue Aug 5 21:08:38 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 5 21:08:38 2014 +0100 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/040ef798/pom.xml -- diff --git a/pom.xml b/pom.xml index 7cca66c..253d40f 100644 --- a/pom.xml +++ b/pom.xml @@ -1264,7 +1264,7 @@ branch-2 -2.5.0-SNAPSHOT +2.6.0-SNAPSHOT
[12/18] git commit: SLIDER-271. When a container is releases the AppMaster should not monitor heartbeats from that container SLIDER-293. On Windows slider agent should be able to execute commands and
SLIDER-271. When a container is releases the AppMaster should not monitor
heartbeats from that container
SLIDER-293. On Windows slider agent should be able to execute commands and not
wait for completion
SLIDER-294. Clean up memcached app packages for Windows and Linux
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/405aa51e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/405aa51e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/405aa51e
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 405aa51e203f611a385a0e923fe68eb0d7d86fc0
Parents: 640233d
Author: Sumit Mohanty
Authored: Tue Aug 5 18:31:16 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached-win/appConfig.json | 6 +-
app-packages/memcached-win/metainfo.xml | 7 +-
.../memcached-win/package/scripts/memcached.py | 2 +-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 6 +-
app-packages/memcached/metainfo.xml | 1 -
.../core/providers/windows/system.py| 54 ++-
.../providers/agent/AgentProviderService.java | 71 +++-
.../agent/TestAgentProviderService.java | 17 -
9 files changed, 121 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/appConfig.json
--
diff --git a/app-packages/memcached-win/appConfig.json
b/app-packages/memcached-win/appConfig.json
index 8c00660..63835ec 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -5,12 +5,16 @@
"global": {
"application.def": "/slider/jmemcached-1.0.0.zip",
"java_home": "C:\\java",
+
"site.global.app_user": "hadoop",
"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
+
"site.global.additional_cp":
"C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
-"site.global.memory_val": "200M"
+"site.global.memory_val": "200M",
+"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/metainfo.xml
--
diff --git a/app-packages/memcached-win/metainfo.xml
b/app-packages/memcached-win/metainfo.xml
index dc2d67b..d056c0a 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -28,10 +28,15 @@
MEMCACHED
MASTER
+
+
+host_port
+${THIS_HOST}:${site.global.listen_port}
+
+
scripts/memcached.py
PYTHON
- 600
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/memcached.py
--
diff --git a/app-packages/memcached-win/package/scripts/memcached.py
b/app-packages/memcached-win/package/scripts/memcached.py
index 222a62c..1bb05dc 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -34,7 +34,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
-process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val}")
+process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/params.py
--
diff --git a/app-packages/memcached-win/package/scripts/params.py
b/app-packages/memcached-win/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp =
config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']
[13/18] git commit: SLIDER-272. Application should specify the config types to publish
SLIDER-272. Application should specify the config types to publish
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/640233dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/640233dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/640233dd
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 640233dd355f8ac3fe917a307ca3251caf6593e7
Parents: 50dc76c
Author: Sumit Mohanty
Authored: Sun Aug 3 21:48:32 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached-win/metainfo.xml | 1 +
app-packages/memcached/metainfo.xml | 1 +
.../providers/agent/AgentProviderService.java| 19 +--
.../agent/application/metadata/Application.java | 9 +
.../application/metadata/MetainfoParser.java | 1 +
.../agent/TestAgentProviderService.java | 2 ++
6 files changed, 31 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached-win/metainfo.xml
--
diff --git a/app-packages/memcached-win/metainfo.xml
b/app-packages/memcached-win/metainfo.xml
index a0de8c2..dc2d67b 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -22,6 +22,7 @@
MEMCACHED
Memcache is a network accessible key/value storage system, often
used as a distributed cache.
1.0.0
+None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached/metainfo.xml
--
diff --git a/app-packages/memcached/metainfo.xml
b/app-packages/memcached/metainfo.xml
index 0550b7d..08a504e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -22,6 +22,7 @@
MEMCACHED
Memcache is a network accessible key/value storage system, often
used as a distributed cache.
1.0.0
+None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 28c874b..682efba 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -88,6 +88,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -637,12 +638,26 @@ public class AgentProviderService extends
AbstractProviderService implements
Application application = getMetainfo().getApplication();
List exportGroups = application.getExportGroups();
boolean hasExportGroups = exportGroups != null &&
!exportGroups.isEmpty();
+ Set exportedConfigs = new HashSet();
+ String exportedConfigsStr = application.getExportedConfigs();
+ boolean exportedConfigSpecified = exportedConfigsStr != null;
+ if (application.getExportedConfigs() != null &&
application.getExportedConfigs().length() > 0) {
+for (String exportedConfig :
application.getExportedConfigs().split(",")) {
+ if (exportedConfig.trim().length() > 0) {
+exportedConfigs.add(exportedConfig.trim());
+ }
+}
+ }
+
for (ComponentStatus status : statuses) {
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
-Map configs = status.getConfigs().get(key);
-publishApplicationInstanceData(key, key, configs.entrySet());
+if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
+!exportedConfigSpecified) {
+ Map configs = status.getConfigs().get(key);
+ publishApplicationInstanceData(key, key, configs.entrySet());
+}
}
if (hasExportGroups) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/pro
git commit: SLIDER-276 : AgentProvider releases nodes that the AM has already been released after detecting heartbeat failure
Repository: incubator-slider
Updated Branches:
refs/heads/develop d68be5aa5 -> 32fd3f56a
SLIDER-276 : AgentProvider releases nodes that the AM has already been released
after detecting heartbeat failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/32fd3f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/32fd3f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/32fd3f56
Branch: refs/heads/develop
Commit: 32fd3f56a785e29b55c0560c6256e50bf07215b8
Parents: d68be5a
Author: Steve Loughran
Authored: Wed Aug 6 14:41:00 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 14:41:00 2014 +0100
--
.../providers/agent/AgentProviderService.java | 34 ++---
.../providers/agent/ComponentCommandOrder.java | 8 +-
.../providers/agent/ComponentInstanceState.java | 80 +++---
.../providers/agent/HeartbeatMonitor.java | 66 +---
.../server/appmaster/AMViewForProviders.java| 21 ++-
.../server/appmaster/SliderAppMaster.java | 27 ++--
.../slider/server/appmaster/state/AppState.java | 6 +-
.../appmaster/model/mock/MockContainer.groovy | 2 +-
.../agent/TestAgentProviderService.java | 151 +++
.../agent/TestComponentCommandOrder.java| 17 ++-
.../agent/TestComponentInstanceState.java | 124 ---
.../providers/agent/TestHeartbeatMonitor.java | 120 ++-
12 files changed, 337 insertions(+), 319 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 6cfadf3..4d54509 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -58,6 +58,7 @@ import
org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -127,10 +128,10 @@ public class AgentProviderService extends
AbstractProviderService implements
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
- private Map componentStatuses = new
ConcurrentHashMap<>();
- private Map> componentInstanceData = new
ConcurrentHashMap();
+ private final Map componentStatuses = new
ConcurrentHashMap<>();
+ private final Map> componentInstanceData = new
ConcurrentHashMap<>();
private final Map> allocatedPorts = new
ConcurrentHashMap<>();
- private Map workFolders =
+ private final Map workFolders =
Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
@@ -291,7 +292,7 @@ public class AgentProviderService extends
AbstractProviderService implements
getComponentStatuses().put(label,
new ComponentInstanceState(
role,
- container.getId().toString(),
+ container.getId(),
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
@@ -333,7 +334,7 @@ public class AgentProviderService extends
AbstractProviderService implements
String label = registration.getHostname();
if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
-
getComponentStatuses().get(label).setLastHeartbeat(System.currentTimeMillis());
+ getComponentStatuses().get(label).heartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
@@ -371,7 +372,7 @@ public class AgentProviderService extends
AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
-compone
[7/8] git commit: Merge branch 'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release' into develop
Merge branch 'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d19c8022 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d19c8022 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d19c8022 Branch: refs/heads/develop Commit: d19c802264afc9121cd9470860d2ac66caf5357f Parents: 32fd3f5 0646c23 Author: Steve Loughran Authored: Wed Aug 6 19:25:17 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 19:25:17 2014 +0100 -- .../apache/slider/common/tools/Comparators.java | 58 + .../providers/AbstractProviderService.java | 29 ++- .../slider/providers/ProviderService.java | 20 +- .../appmaster/AsyncRMOperationHandler.java | 51 .../server/appmaster/SliderAppMaster.java | 19 +- .../operations/AbstractRMOperation.java | 29 +++ .../operations/AsyncRMOperationHandler.java | 51 .../operations/ContainerReleaseOperation.java | 44 .../operations/ContainerRequestOperation.java | 44 .../ProviderNotifyingOperationHandler.java | 43 .../operations/RMOperationHandler.java | 35 +++ .../operations/RMOperationHandlerActions.java | 28 ++ .../appmaster/state/AbstractRMOperation.java| 31 --- .../slider/server/appmaster/state/AppState.java | 135 +++--- .../state/ContainerReleaseOperation.java| 44 .../state/ContainerReleaseSelector.java | 38 +++ .../state/ContainerRequestOperation.java| 44 .../MostRecentContainerReleaseSelector.java | 52 .../server/appmaster/state/NodeInstance.java| 12 +- .../slider/server/appmaster/state/NodeMap.java | 54 .../appmaster/state/RMOperationHandler.java | 43 .../server/appmaster/state/RoleHistory.java | 14 +- .../appmaster/state/SimpleReleaseSelector.java | 34 +++ .../agent/TestAgentAMManagementWS.groovy| 11 +- .../TestAppStateContainerFailure.groovy | 166 .../appstate/TestAppStateDynamicRoles.groovy| 93 --- .../TestAppStateRebuildOnAMRestart.groovy | 124 - .../appstate/TestAppStateRolePlacement.groovy | 99 --- .../appstate/TestAppStateRoleRelease.groovy | 82 -- .../TestContainerResourceAllocations.groovy | 108 .../model/appstate/TestFlexDynamicRoles.groovy | 189 -- .../TestMockAppStateContainerFailure.groovy | 166 .../TestMockAppStateDynamicRoles.groovy | 94 +++ .../TestMockAppStateFlexDynamicRoles.groovy | 190 ++ .../appstate/TestMockAppStateFlexing.groovy | 116 + .../TestMockAppStateRMOperations.groovy | 180 + .../TestMockAppStateRebuildOnAMRestart.groovy | 125 + .../TestMockAppStateRolePlacement.groovy| 102 .../appstate/TestMockAppStateRoleRelease.groovy | 82 ++ .../TestMockContainerResourceAllocations.groovy | 108 .../model/appstate/TestMockFlexing.groovy | 128 - .../model/appstate/TestMockRMOperations.groovy | 176 - .../history/TestFindNodesForNewInstances.groovy | 128 - .../history/TestFindNodesForRelease.groovy | 146 --- .../model/history/TestHistoryRW.groovy | 258 --- .../model/history/TestHistoryRWOrdering.groovy | 145 --- .../model/history/TestNIComparators.groovy | 77 -- .../TestOutstandingRequestTracker.groovy| 73 -- .../TestRoleHistoryContainerEvents.groovy | 17 +- ...stRoleHistoryFindNodesForNewInstances.groovy | 128 + .../history/TestRoleHistoryNIComparators.groovy | 77 ++ ...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++ .../model/history/TestRoleHistoryRW.groovy | 258 +++ .../history/TestRoleHistoryRWOrdering.groovy| 145 +++ .../model/mock/BaseMockAppStateTest.groovy | 3 +- .../model/mock/MockProviderService.groovy | 69 +++-- .../model/mock/MockRMOperationHandler.groovy| 8 +- .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 3 +- 60 files changed, 2544 insertions(+), 2364 deletions(-) --
[8/8] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ec760682 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ec760682 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ec760682 Branch: refs/heads/develop Commit: ec76068292dd5d9c53ce3983dbbbdea1bf793558 Parents: d19c802 533b229 Author: Steve Loughran Authored: Wed Aug 6 19:25:24 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 19:25:24 2014 +0100 -- .../java/org/apache/slider/server/appmaster/state/AppState.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ec760682/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java --
[6/8] git commit: SLIDER-296 providers now get notified of container create/release requests
SLIDER-296 providers now get notified of container create/release requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0646c234
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0646c234
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0646c234
Branch: refs/heads/develop
Commit: 0646c234ff20a98c4620f2dbd69a4a36414ca27d
Parents: 868f7e2
Author: Steve Loughran
Authored: Wed Aug 6 19:24:55 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 19:24:55 2014 +0100
--
.../providers/AbstractProviderService.java | 11 +
.../slider/providers/ProviderService.java | 8 +--
.../appmaster/AsyncRMOperationHandler.java | 51
.../server/appmaster/SliderAppMaster.java | 15 --
.../operations/AbstractRMOperation.java | 29 +++
.../operations/AsyncRMOperationHandler.java | 51
.../operations/ContainerReleaseOperation.java | 44 +
.../operations/ContainerRequestOperation.java | 44 +
.../ProviderNotifyingOperationHandler.java | 43 +
.../operations/RMOperationHandler.java | 35 ++
.../operations/RMOperationHandlerActions.java | 28 +++
.../appmaster/state/AbstractRMOperation.java| 31
.../slider/server/appmaster/state/AppState.java | 5 +-
.../state/ContainerReleaseOperation.java| 44 -
.../state/ContainerRequestOperation.java| 44 -
.../appmaster/state/RMOperationHandler.java | 43 -
.../TestMockAppStateDynamicRoles.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 12 +
.../TestMockAppStateRMOperations.groovy | 4 ++
.../TestMockAppStateRebuildOnAMRestart.groovy | 1 +
.../TestMockAppStateRolePlacement.groovy| 3 ++
.../appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../TestMockContainerResourceAllocations.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 1 +
.../model/mock/MockProviderService.groovy | 11 +
.../model/mock/MockRMOperationHandler.groovy| 8 +--
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +--
27 files changed, 337 insertions(+), 243 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7483a8d..158640d 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -337,4 +338,14 @@ public abstract class AbstractProviderService
public ContainerReleaseSelector createContainerReleaseSelector() {
return new MostRecentContainerReleaseSelector();
}
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+// no-op
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+// no-op
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4278b09..c0a75c1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -34,18 +34,20 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
+import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.
[5/8] git commit: IDE-indicated import cleanup
IDE-indicated import cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/868f7e2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/868f7e2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/868f7e2c
Branch: refs/heads/develop
Commit: 868f7e2c6319040d4f9edea202672f18b7939f1a
Parents: 3bb202f
Author: Steve Loughran
Authored: Wed Aug 6 17:29:18 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:29:18 2014 +0100
--
.../providers/agent/TestAgentAMManagementWS.groovy | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/868f7e2c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index d3a0be7..e248ec3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -27,7 +27,6 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
import org.apache.slider.server.services.security.CertificateManager
@@ -37,13 +36,15 @@ import org.junit.Test
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import javax.net.ssl.HostnameVerifier
+import javax.net.ssl.HttpsURLConnection
+import javax.net.ssl.SSLSession
import javax.ws.rs.core.MediaType
import static org.apache.slider.common.params.Arguments.ARG_OPTION
import static org.apache.slider.providers.agent.AgentKeys.*
import static
org.apache.slider.providers.agent.AgentTestUtils.createDummyJSONRegister
import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
-import static org.apache.slider.test.SliderTestUtils.log
@CompileStatic
@Slf4j
@@ -53,10 +54,10 @@ class TestAgentAMManagementWS extends AgentTestBase {
final static Logger logger =
LoggerFactory.getLogger(TestAgentAMManagementWS.class)
static {
//for localhost testing only
-javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
-new javax.net.ssl.HostnameVerifier(){
+HttpsURLConnection.setDefaultHostnameVerifier(
+new HostnameVerifier(){
public boolean verify(String hostname,
- javax.net.ssl.SSLSession sslSession)
{
+ SSLSession sslSession) {
logger.info("verifying hostname ${hostname}")
InetAddress[] addresses =
InetAddress.getAllByName(hostname);
[1/8] SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
Repository: incubator-slider
Updated Branches:
refs/heads/develop 533b229f0 -> ec7606829
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
new file mode 100644
index 000..6e18e39
--- /dev/null
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static
org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static
org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockAppStateRMOperations extends BaseMockAppStateTest implements
MockRoles {
+
+ @Override
+ String getTestName() {
+return "TestMockRMOperations"
+ }
+
+ @Test
+ public void testPriorityOnly() throws Throwable {
+assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTrip() throws Throwable {
+assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTripWithRequest() throws Throwable {
+int priority = buildPriority(5, false)
+assert 5 == extractRole(priority)
+ }
+
+ @Test
+ public void testMockAddOp() throws Throwable {
+role0Status.desired = 1
+List ops = appState.reviewRequestAndReleaseNodes()
+assert ops.size() == 1
+ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+int priority = operation.request.priority.priority
+assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
+RMOperationHandler handler = new MockRMOperationHandler()
+handler.execute(ops)
+
+//tell the container its been allocated
+AbstractRMOperation op = handler.operations[0]
+assert op instanceof ContainerRequestOperation
+ }
+
+ @Test
+ public void testAllocateReleaseOp() throws Throwable {
+role0Status.desired = 1
+
+List ops = appState.reviewRequestAndReleaseNodes()
+ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+AMRMClient.ContainerRequest request = operation.request
+Container cont = engine.allocateContainer(request)
+List allocated = [cont]
+List assignments = [];
+List operations = []
+appState.onContainersAllocated(allocated, assignments, operations)
+assert operations.size() == 0
+assert assignments.size() == 1
+ContainerAssignment assigned = assignments[0]
+Container target = assigned.container
+assert target.id == cont.id
+int roleId = assigned.role.priority
+assert roleId == extractRole(request.priority)
+assert assigned.role.name == ROLE0
+RoleInstance ri = roleInstance(assigned)
+//tell the app it arrived
+appState.containerStartSubmitted(target, ri);
+appState.innerOnNodeManagerContainerStarted(target.id)
+assert role0Status.started == 1
+
+//now release it by changing the role status
+role0Status.desired = 0
+ops = appState.reviewRequestAndReleaseNodes()
+assert ops.size() == 1
+
+assert ops[0] instanceof ContainerReleaseOperation
+ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+assert release.containerId == cont.id
+ }
+
+ @Test
+ public void
[4/8] git commit: SLIDER-296 removing use of RoleHistory/nodemap target release selection logic - cutting methods out entirely
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic -
cutting methods out entirely
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3bb202f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3bb202f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3bb202f5
Branch: refs/heads/develop
Commit: 3bb202f54ce74541a7926b04555732326df7f5d0
Parents: 8cecce1
Author: Steve Loughran
Authored: Wed Aug 6 17:13:40 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:13:40 2014 +0100
--
.../slider/server/appmaster/state/NodeMap.java | 54 ---
.../server/appmaster/state/RoleHistory.java | 16 --
.../TestRoleHistoryFindNodesForRelease.groovy | 146 ---
3 files changed, 216 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 32b1656..372a0a5 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -102,60 +102,6 @@ public class NodeMap extends HashMap
{
}
return purged;
}
-
-
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public List findNodesForRelease(int role, int count) {
-List targets = new ArrayList<>(count);
-List active = listActiveNodes(role);
-List multiple = new ArrayList<>();
-int nodesRemaining = count;
-log.debug("searching for {} nodes with candidate set size {}",
- nodesRemaining, active.size());
-ListIterator it = active.listIterator();
-while (it.hasNext() && nodesRemaining > 0) {
- NodeInstance ni = it.next();
- int load = ni.getActiveRoleInstances(role);
- log.debug("Node {} load={}", ni, load);
- assert load != 0;
- if (load == 1) {
-// at the tail of the list, from here active[*] is a load=1 entry
-break;
- }
- // load is >1. Add an entry to the target list FOR EACH INSTANCE ABOVE 1
- for (int i = 0; i < (load - 1) && nodesRemaining > 0; i++) {
-nodesRemaining--;
-log.debug("Push {} #{}", ni, i);
-targets.add(ni);
- }
- // and add to the multiple list
- multiple.add(ni);
- // then pop it from the active list
- it.remove();
-}
-//here either the number is found or there is still some left.
-
-if (nodesRemaining > 0) {
- // leftovers. Append any of the multiple node entries to the tail of
- // the active list (so they get chosen last)
- active.addAll(multiple);
- // all the entries in the list have exactly one node
- // so ask for as many as are needed
- int ask = Math.min(nodesRemaining, active.size());
- log.debug("load=1 nodes to select={} multiples={} available={} ask={}",
-nodesRemaining, multiple.size(),active.size(), ask);
- targets.addAll(active.subList(0, ask));
-}
-return targets;
- }
-
/**
* Clone point
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index dbb104c..e9da081 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -517,19 +517,6 @@ public class RoleHistory {
return requestInstanceOnNode(node, role, resource);
}
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public synchronized List findNodesForRelease(int role, int
count) {
-return nodemap.findNodesForRelease(role, count);
- }
-
-
/**
* Get the list of active nodes ... walks the node map so
* is O(nodes)
@@ -539,9 +526,6 @@ public class RoleHistory {
public syn
[3/8] git commit: SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8cecce18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8cecce18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8cecce18
Branch: refs/heads/develop
Commit: 8cecce1821733c33de3cf9723f9a96aee411305d
Parents: 5ce953c
Author: Steve Loughran
Authored: Wed Aug 6 17:11:39 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:11:39 2014 +0100
--
.../server/appmaster/state/NodeInstance.java| 12 +-
.../server/appmaster/state/RoleHistory.java | 14 +
.../history/TestFindNodesForNewInstances.groovy | 128 -
.../history/TestFindNodesForRelease.groovy | 146 ---
.../model/history/TestHistoryRW.groovy | 258 ---
.../model/history/TestHistoryRWOrdering.groovy | 145 ---
.../model/history/TestNIComparators.groovy | 77 --
.../TestOutstandingRequestTracker.groovy| 73 --
.../TestRoleHistoryContainerEvents.groovy | 17 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 +
.../TestRoleHistoryFindNodesForRelease.groovy | 146 +++
.../history/TestRoleHistoryNIComparators.groovy | 77 ++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++
.../model/history/TestRoleHistoryRW.groovy | 258 +++
.../history/TestRoleHistoryRWOrdering.groovy| 145 +++
15 files changed, 861 insertions(+), 836 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 06375fb..fad60f2 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -75,7 +75,7 @@ public class NodeInstance {
}
/**
- * Cout the number of active role instances on this node
+ * Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
* not being released already.
@@ -84,6 +84,16 @@ public class NodeInstance {
NodeEntry nodeEntry = get(role);
return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
}
+
+ /**
+ * Count the number of live role instances on this node
+ * @param role role index
+ * @return 0 if there are none, otherwise the #of nodes that are running
+ */
+ public int getLiveRoleInstances(int role) {
+NodeEntry nodeEntry = get(role);
+return (nodeEntry != null ) ? nodeEntry.getLive() : 0;
+ }
/**
* Get the entry for a role -and remove it if present
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index de7e9d8..dbb104c 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -529,6 +529,20 @@ public class RoleHistory {
return nodemap.findNodesForRelease(role, count);
}
+
+ /**
+ * Get the list of active nodes ... walks the node map so
+ * is O(nodes)
+ * @param role role index
+ * @return a possibly empty list of nodes with an instance of that node
+ */
+ public synchronized List listActiveNodes(int role) {
+return nodemap.listActiveNodes(role);
+ }
+
+
+
+
/**
* Get the node entry of a container
* @param container container to look up
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
deleted file mode 100644
index dab03f5..000
---
a/slider-core/src/test
[2/8] git commit: SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to
release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ce953ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ce953ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ce953ca
Branch: refs/heads/develop
Commit: 5ce953ca301a90e27b1cbe193a482a9cb83c3084
Parents: 32fd3f5
Author: Steve Loughran
Authored: Wed Aug 6 16:58:27 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 16:58:27 2014 +0100
--
.../apache/slider/common/tools/Comparators.java | 58 ++
.../providers/AbstractProviderService.java | 18 +-
.../slider/providers/ProviderService.java | 12 +-
.../server/appmaster/SliderAppMaster.java | 4 +-
.../slider/server/appmaster/state/AppState.java | 134 +
.../state/ContainerReleaseSelector.java | 38
.../MostRecentContainerReleaseSelector.java | 52 +
.../appmaster/state/SimpleReleaseSelector.java | 34
.../TestAppStateContainerFailure.groovy | 166
.../appstate/TestAppStateDynamicRoles.groovy| 93 -
.../TestAppStateRebuildOnAMRestart.groovy | 124
.../appstate/TestAppStateRolePlacement.groovy | 99 --
.../appstate/TestAppStateRoleRelease.groovy | 82
.../TestContainerResourceAllocations.groovy | 108 ---
.../model/appstate/TestFlexDynamicRoles.groovy | 189 --
.../TestMockAppStateContainerFailure.groovy | 166
.../TestMockAppStateDynamicRoles.groovy | 94 +
.../TestMockAppStateFlexDynamicRoles.groovy | 190 +++
.../appstate/TestMockAppStateFlexing.groovy | 126
.../TestMockAppStateRMOperations.groovy | 176 +
.../TestMockAppStateRebuildOnAMRestart.groovy | 124
.../TestMockAppStateRolePlacement.groovy| 99 ++
.../appstate/TestMockAppStateRoleRelease.groovy | 82
.../TestMockContainerResourceAllocations.groovy | 108 +++
.../model/appstate/TestMockFlexing.groovy | 128 -
.../model/appstate/TestMockRMOperations.groovy | 176 -
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 56 +++---
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 3 +-
30 files changed, 1510 insertions(+), 1234 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 000..0ccca0f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+public class Comparators {
+
+ public static class LongComparator implements Comparator, Serializable
{
+@Override
+public int compare(Long o1, Long o2) {
+ long result = o1 - o2;
+ // need to comparisons with a diff greater than integer size
+ if (result < 0 ) {
+return -1;
+ } else if (result >0) {
+return 1;
+ }
+ return 0;
+}
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param the type that is being compared
+ */
+ public static class ComparatorReverser implements
Comparator,
+ Serializable {
+
+final Comparator instance;
+
+public ComparatorReverser(Comparator instance) {
+ this.instance = instance;
+}
+
+@Override
+publ
[4/5] git commit: SLIDER-286 adds the live container list as an argument in the bind() operation...all implementations currently discard this.
SLIDER-286 adds the live container list as an argument in the bind()
operation...all implementations currently discard this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/11335c1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/11335c1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/11335c1b
Branch: refs/heads/develop
Commit: 11335c1be7f41a27354417e15ed0aa83829684f2
Parents: 1350491
Author: Steve Loughran
Authored: Thu Aug 7 17:37:59 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:37:59 2014 +0100
--
.../slider/providers/AbstractProviderService.java | 6 --
.../org/apache/slider/providers/ProviderService.java | 14 +-
.../slider/server/appmaster/SliderAppMaster.java | 13 -
.../appmaster/model/mock/MockProviderService.groovy | 3 ++-
.../web/rest/publisher/TestAgentProviderService.java | 10 +++---
5 files changed, 30 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 158640d..b431a10 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
@@ -46,7 +47,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -97,7 +97,9 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
+ RegistryViewForProviders reg,
+ AMViewForProviders amView,
+ List liveContainers) {
this.amState = stateAccessor;
this.registry = reg;
this.amView = amView;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index c0a75c1..1466350 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -42,6 +42,7 @@ import
org.apache.slider.server.services.registry.RegistryViewForProviders;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.List;
import java.util.Map;
public interface ProviderService extends ProviderCore,
@@ -74,7 +75,7 @@ public interface ProviderService extends ProviderCore,
/**
* Notify the providers of container completion
- * @param containerId
+ * @param containerId container that has completed
*/
void notifyContainerCompleted(ContainerId containerId);
@@ -148,12 +149,15 @@ public interface ProviderService extends ProviderCore,
/**
* bind operation -invoked before the service is started
* @param stateAccessor interface offering read access to the state
- * @param registry
- * @param amView
+ * @param registry registry view
+ * @param amView AM access
+ * @param liveContainers list of live containers. If non-empty, the AM was
+ * restarted with one more running containers.
*/
void bind(StateAccessForProviders stateAccessor,
-RegistryViewForProviders registry,
-AMViewForProviders amView);
+ RegistryViewForProviders registry,
+ AMViewForProviders amView,
+ List liveContainers);
/**
* Returns the agent rest operations interface.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppM
[3/5] git commit: SLIDER-286 AM restart count is now set to 0 in AppMasterLauncher/Slider client, so AMs get default cluster value. The value is logged at debug level
SLIDER-286 AM restart count is now set to 0 in AppMasterLauncher/Slider client,
so AMs get default cluster value. The value is logged at debug level
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1350491f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1350491f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1350491f
Branch: refs/heads/develop
Commit: 1350491fd487987b458da75dc571c07c1ae76770
Parents: 56083ec
Author: Steve Loughran
Authored: Thu Aug 7 17:36:45 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:36:45 2014 +0100
--
.../src/main/java/org/apache/slider/client/SliderClient.java | 4 ++--
.../java/org/apache/slider/core/launch/AppMasterLauncher.java | 7 +--
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ff2e865..e1d083c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -833,8 +833,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
-amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT,
- DEFAULT_AM_RESTART_LIMIT));
+int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
+amLauncher.setMaxAppAttempts(maxAppAttempts);
sliderFileSystem.purgeAppInstanceTempFiles(clustername);
Path tempPath = sliderFileSystem.createAppInstanceTempPath(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index dc4a886..bd8a0a5 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -50,7 +50,7 @@ public class AppMasterLauncher extends AbstractLauncher {
private final ApplicationSubmissionContext submissionContext;
private final ApplicationId appId;
private final boolean secureCluster;
- private int maxAppAttempts = 2;
+ private int maxAppAttempts = 0;
private boolean keepContainersOverRestarts = true;
private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private int priority = 1;
@@ -174,7 +174,10 @@ public class AppMasterLauncher extends AbstractLauncher {
submissionContext.setKeepContainersAcrossApplicationAttempts(true);
}
-submissionContext.setMaxAppAttempts(maxAppAttempts);
+if (maxAppAttempts > 0) {
+ log.debug("Setting max AM attempts to {}", maxAppAttempts);
+ submissionContext.setMaxAppAttempts(maxAppAttempts);
+}
if (secureCluster) {
addSecurityTokens();
[1/5] git commit: SLIDER-286 new test for AM restart, and some renaming of methods to make function clearer.
Repository: incubator-slider
Updated Branches:
refs/heads/develop 373aed3d6 -> bd69a4cbb
SLIDER-286 new test for AM restart, and some renaming of methods to make
function clearer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cbd890f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cbd890f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cbd890f
Branch: refs/heads/develop
Commit: 1cbd890fcc805fe2a7ccf9468201d4ecb4c6caeb
Parents: 373aed3
Author: Steve Loughran
Authored: Thu Aug 7 17:26:48 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:26:48 2014 +0100
--
.../agent/AgentMiniClusterTestBase.groovy | 7 +-
.../agent/actions/TestActionExists.groovy | 7 +-
.../slider/agent/actions/TestActionList.groovy | 5 +-
.../agent/actions/TestActionStatus.groovy | 7 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 6 +-
.../TestFreezeThawMasterlessAM.groovy | 5 +-
.../agent/standalone/TestBadAMHeap.groovy | 2 +-
.../agent/standalone/TestBadYarnQueue.groovy| 2 +-
.../standalone/TestDestroyMasterlessAM.groovy | 153 --
.../standalone/TestDestroyStandaloneAM.groovy | 156 +++
.../standalone/TestKillMasterlessAM.groovy | 73 -
.../standalone/TestKillStandaloneAM.groovy | 73 +
.../standalone/TestRestartStandaloneAM.groovy | 96
.../standalone/TestStandaloneAgentAM.groovy | 6 +-
.../TestStandaloneBadClusterName.groovy | 2 +-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
17 files changed, 358 insertions(+), 251 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 8a4e5d8..74f7a3f 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -133,13 +133,12 @@ extends YarnZKMiniClusterTestBase {
* @param blockUntilRunning block until the AM is running
* @return launcher which will have executed the command.
*/
- public ServiceLauncher createMasterlessAM(
+ public ServiceLauncher createStandaloneAM(
String clustername,
- int size,
boolean deleteExistingData,
boolean blockUntilRunning) {
List args = [];
-return createMasterlessAMWithArgs(
+return createStandaloneAMWithArgs(
clustername,
args,
deleteExistingData,
@@ -155,7 +154,7 @@ extends YarnZKMiniClusterTestBase {
* @param blockUntilRunning block until the AM is running
* @return launcher which will have executed the command.
*/
- public ServiceLauncher createMasterlessAMWithArgs(
+ public ServiceLauncher createStandaloneAMWithArgs(
String clustername,
List extraArgs,
boolean deleteExistingData,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index ee8baff..9bfeb8c 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -29,8 +29,6 @@ import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.test.SliderTestUtils
-import org.junit.Assert
import org.junit.Before
import org.junit.Test
@@ -72,7 +70,10 @@ class TestActionExists extends AgentMiniClusterTestBase {
public void testExistsLiveCluster() throws Throwable {
//launch the cluster
String clustername = createClusterName()
-ServiceLauncher launcher = createMasterlessAM(clustername,
0, true, false)
+ServiceLauncher launcher = createStandaloneAM(
+clustername,
+true,
+false)
SliderClient sliderClient = launcher.service
addToTeardown(launcher)
ApplicationRe
[5/5] git commit: Merge branch 'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop
Merge branch 'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd69a4cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd69a4cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd69a4cb Branch: refs/heads/develop Commit: bd69a4cbb71591fc4304e452a5ce036e4cddc6d5 Parents: 373aed3 11335c1 Author: Steve Loughran Authored: Thu Aug 7 17:58:43 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 7 17:58:43 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 4 +- .../slider/core/launch/AppMasterLauncher.java | 7 +- .../providers/AbstractProviderService.java | 6 +- .../slider/providers/ProviderService.java | 14 +- .../server/appmaster/SliderAppMaster.java | 13 +- .../agent/AgentMiniClusterTestBase.groovy | 7 +- .../agent/actions/TestActionExists.groovy | 7 +- .../slider/agent/actions/TestActionList.groovy | 5 +- .../agent/actions/TestActionStatus.groovy | 7 +- .../agent/freezethaw/TestFreezeCommands.groovy | 6 +- .../TestFreezeThawMasterlessAM.groovy | 5 +- .../agent/standalone/TestBadAMHeap.groovy | 2 +- .../agent/standalone/TestBadYarnQueue.groovy| 2 +- .../standalone/TestDestroyMasterlessAM.groovy | 153 -- .../standalone/TestDestroyStandaloneAM.groovy | 156 +++ .../standalone/TestKillMasterlessAM.groovy | 73 - .../standalone/TestKillStandaloneAM.groovy | 73 + .../standalone/TestRestartStandaloneAM.groovy | 96 .../standalone/TestStandaloneAgentAM.groovy | 6 +- .../TestStandaloneBadClusterName.groovy | 2 +- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../model/mock/MockProviderService.groovy | 3 +- .../apache/slider/test/SliderTestUtils.groovy | 2 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../publisher/TestAgentProviderService.java | 10 +- 25 files changed, 396 insertions(+), 272 deletions(-) --
[2/5] git commit: SLIDER-286 bug fix in waitUntilClusterLive
SLIDER-286 bug fix in waitUntilClusterLive
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56083ecc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56083ecc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56083ecc
Branch: refs/heads/develop
Commit: 56083eccf49c4caf78f69d52e33c36bbe3754ba4
Parents: 1cbd890
Author: Steve Loughran
Authored: Thu Aug 7 17:27:13 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:27:13 2014 +0100
--
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56083ecc/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index a250e55..c5a8b36 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -205,7 +205,7 @@ class SliderTestUtils extends Assert {
public static void waitUntilClusterLive(SliderClient client, int timeout) {
Duration duration = new Duration(timeout);
duration.start()
-while (!client.actionExists(client.deployedClusterName, true) &&
+while (0 != client.actionExists(client.deployedClusterName, true) &&
!duration.limitExceeded) {
sleep(1000);
}
[33/38] git commit: SLIDER-286 bug fix in waitUntilClusterLive
SLIDER-286 bug fix in waitUntilClusterLive
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56083ecc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56083ecc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56083ecc
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 56083eccf49c4caf78f69d52e33c36bbe3754ba4
Parents: 1cbd890
Author: Steve Loughran
Authored: Thu Aug 7 17:27:13 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:27:13 2014 +0100
--
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56083ecc/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index a250e55..c5a8b36 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -205,7 +205,7 @@ class SliderTestUtils extends Assert {
public static void waitUntilClusterLive(SliderClient client, int timeout) {
Duration duration = new Duration(timeout);
duration.start()
-while (!client.actionExists(client.deployedClusterName, true) &&
+while (0 != client.actionExists(client.deployedClusterName, true) &&
!duration.limitExceeded) {
sleep(1000);
}
[07/38] git commit: SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex tests
SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex
tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4d0734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4d0734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4d0734
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ae4d07340b0c0fe54b3dc75e58744393d5949d31
Parents: 34c00e8
Author: Steve Loughran
Authored: Tue Aug 5 21:06:54 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:06:54 2014 +0100
--
.../TestClusterFlexHMasterFlex1To2.groovy | 45
.../flexing/TestHMasterFlex1To2.groovy | 45
2 files changed, 45 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
new file mode 100644
index 000..298f40f
--- /dev/null
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterMasterFlex1To2() throws Throwable {
+assert flexHBaseClusterTestRun(
+"",
+1,
+2,
+1,
+1,
+true)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
deleted file mode 100644
index ca26572..000
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.minicluster.flexing
[02/38] git commit: SLIDER-265 Add role for HBase REST gateway (Sumit) - add hbase_rest.py
SLIDER-265 Add role for HBase REST gateway (Sumit) - add hbase_rest.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ab9ce96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ab9ce96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ab9ce96
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7ab9ce96e8220a8f2a8b212d798306b5bdd24dc0
Parents: 94263fc
Author: tedyu
Authored: Mon Aug 4 13:13:09 2014 -0700
Committer: tedyu
Committed: Mon Aug 4 13:13:09 2014 -0700
--
.../hbase/package/scripts/hbase_rest.py | 62
1 file changed, 62 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ab9ce96/app-packages/hbase/package/scripts/hbase_rest.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_rest.py
b/app-packages/hbase/package/scripts/hbase_rest.py
new file mode 100644
index 000..36b51f9
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_rest.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRest(Script):
+ def install(self, env):
+self.install_packages(env)
+
+ def configure(self, env):
+import params
+env.set_params(params)
+
+hbase(name='rest')
+
+ def start(self, env):
+import params
+env.set_params(params)
+self.configure(env) # for security
+
+hbase_service( 'rest',
+ action = 'start'
+)
+
+ def stop(self, env):
+import params
+env.set_params(params)
+
+hbase_service( 'rest',
+ action = 'stop'
+)
+
+ def status(self, env):
+import status_params
+env.set_params(status_params)
+pid_file = format("{pid_dir}/hbase-{hbase_user}-rest.pid")
+check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseRest().execute()
[35/38] git commit: SLIDER-286 adds the live container list as an argument in the bind() operation...all implementations currently discard this.
SLIDER-286 adds the live container list as an argument in the bind()
operation...all implementations currently discard this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/11335c1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/11335c1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/11335c1b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 11335c1be7f41a27354417e15ed0aa83829684f2
Parents: 1350491
Author: Steve Loughran
Authored: Thu Aug 7 17:37:59 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:37:59 2014 +0100
--
.../slider/providers/AbstractProviderService.java | 6 --
.../org/apache/slider/providers/ProviderService.java | 14 +-
.../slider/server/appmaster/SliderAppMaster.java | 13 -
.../appmaster/model/mock/MockProviderService.groovy | 3 ++-
.../web/rest/publisher/TestAgentProviderService.java | 10 +++---
5 files changed, 30 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 158640d..b431a10 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
@@ -46,7 +47,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -97,7 +97,9 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
+ RegistryViewForProviders reg,
+ AMViewForProviders amView,
+ List liveContainers) {
this.amState = stateAccessor;
this.registry = reg;
this.amView = amView;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index c0a75c1..1466350 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -42,6 +42,7 @@ import
org.apache.slider.server.services.registry.RegistryViewForProviders;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.List;
import java.util.Map;
public interface ProviderService extends ProviderCore,
@@ -74,7 +75,7 @@ public interface ProviderService extends ProviderCore,
/**
* Notify the providers of container completion
- * @param containerId
+ * @param containerId container that has completed
*/
void notifyContainerCompleted(ContainerId containerId);
@@ -148,12 +149,15 @@ public interface ProviderService extends ProviderCore,
/**
* bind operation -invoked before the service is started
* @param stateAccessor interface offering read access to the state
- * @param registry
- * @param amView
+ * @param registry registry view
+ * @param amView AM access
+ * @param liveContainers list of live containers. If non-empty, the AM was
+ * restarted with one more running containers.
*/
void bind(StateAccessForProviders stateAccessor,
-RegistryViewForProviders registry,
-AMViewForProviders amView);
+ RegistryViewForProviders registry,
+ AMViewForProviders amView,
+ List liveContainers);
/**
* Returns the agent rest operations interface.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/or
[09/38] git commit: SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT
SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/040ef798 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/040ef798 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/040ef798 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 040ef79802d0f9c3a77a4b64a5fbea0d05f67fb0 Parents: 268253c Author: Steve Loughran Authored: Tue Aug 5 21:08:38 2014 +0100 Committer: Steve Loughran Committed: Tue Aug 5 21:08:38 2014 +0100 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/040ef798/pom.xml -- diff --git a/pom.xml b/pom.xml index 7cca66c..253d40f 100644 --- a/pom.xml +++ b/pom.xml @@ -1264,7 +1264,7 @@ branch-2 -2.5.0-SNAPSHOT +2.6.0-SNAPSHOT
[36/38] git commit: Merge branch 'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop
Merge branch 'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd69a4cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd69a4cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd69a4cb Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: bd69a4cbb71591fc4304e452a5ce036e4cddc6d5 Parents: 373aed3 11335c1 Author: Steve Loughran Authored: Thu Aug 7 17:58:43 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 7 17:58:43 2014 +0100 -- .../org/apache/slider/client/SliderClient.java | 4 +- .../slider/core/launch/AppMasterLauncher.java | 7 +- .../providers/AbstractProviderService.java | 6 +- .../slider/providers/ProviderService.java | 14 +- .../server/appmaster/SliderAppMaster.java | 13 +- .../agent/AgentMiniClusterTestBase.groovy | 7 +- .../agent/actions/TestActionExists.groovy | 7 +- .../slider/agent/actions/TestActionList.groovy | 5 +- .../agent/actions/TestActionStatus.groovy | 7 +- .../agent/freezethaw/TestFreezeCommands.groovy | 6 +- .../TestFreezeThawMasterlessAM.groovy | 5 +- .../agent/standalone/TestBadAMHeap.groovy | 2 +- .../agent/standalone/TestBadYarnQueue.groovy| 2 +- .../standalone/TestDestroyMasterlessAM.groovy | 153 -- .../standalone/TestDestroyStandaloneAM.groovy | 156 +++ .../standalone/TestKillMasterlessAM.groovy | 73 - .../standalone/TestKillStandaloneAM.groovy | 73 + .../standalone/TestRestartStandaloneAM.groovy | 96 .../standalone/TestStandaloneAgentAM.groovy | 6 +- .../TestStandaloneBadClusterName.groovy | 2 +- .../standalone/TestStandaloneRegistryAM.groovy | 2 +- .../model/mock/MockProviderService.groovy | 3 +- .../apache/slider/test/SliderTestUtils.groovy | 2 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../publisher/TestAgentProviderService.java | 10 +- 25 files changed, 396 insertions(+), 272 deletions(-) --
[31/38] git commit: SLIDER-298. Add an application package template
SLIDER-298. Add an application package template Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/373aed3d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/373aed3d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/373aed3d Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 373aed3d61744342f76b7a859d3660fc52cc6709 Parents: 609fcd3 Author: Sumit Mohanty Authored: Wed Aug 6 22:40:01 2014 -0700 Committer: Sumit Mohanty Committed: Wed Aug 6 22:40:01 2014 -0700 -- app-packages/app-pkg-template/myapp-1.0.0.zip | Bin 5455 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/373aed3d/app-packages/app-pkg-template/myapp-1.0.0.zip -- diff --git a/app-packages/app-pkg-template/myapp-1.0.0.zip b/app-packages/app-pkg-template/myapp-1.0.0.zip deleted file mode 100644 index 80dbbb7..000 Binary files a/app-packages/app-pkg-template/myapp-1.0.0.zip and /dev/null differ
[06/38] git commit: SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 18:10:59 2014 +0100
--
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ---
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58
3 files changed, 151 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
-extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY
<< YRAM
-extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
[:])
}
+ public List hbaseArgs(List extraArgs) {
+extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+YRAM
+extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
-[
-ARG_PROVIDER, PROVIDER_HBASE
-],
+hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
-createMiniCluster(clustername, configuration,
- 1,
- true);
+SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
+
+//now flex
+return flexCluster(
+sliderClient,
+clustername,
+masterFlexTarget,
+workerFlexTarget,
+testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+clustername = createMiniCluster(clustername, configuration,
+1,
+true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true
[29/38] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ec760682 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ec760682 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ec760682 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: ec76068292dd5d9c53ce3983dbbbdea1bf793558 Parents: d19c802 533b229 Author: Steve Loughran Authored: Wed Aug 6 19:25:24 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 19:25:24 2014 +0100 -- .../java/org/apache/slider/server/appmaster/state/AppState.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ec760682/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java --
[37/38] git commit: SLIDER-240 Detect discrepancy between file content of {application.def} and {package_list} value
SLIDER-240 Detect discrepancy between file content of {application.def} and
{package_list} value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66cbd376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66cbd376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66cbd376
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 66cbd3761ead0fa39132b2a07833c9e6b2f4642d
Parents: bd69a4c
Author: tedyu
Authored: Thu Aug 7 11:41:23 2014 -0700
Committer: tedyu
Committed: Thu Aug 7 11:41:23 2014 -0700
--
.../slider/common/tools/CoreFileSystem.java | 33
.../providers/agent/AgentClientProvider.java| 6
2 files changed, 39 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index cada732..93de4d9 100644
---
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -48,8 +48,11 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import static
org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static
org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
@@ -304,6 +307,36 @@ public class CoreFileSystem {
}
/**
+ * Verify that a file exists in the zip file given by path
+ * @param path path to zip file
+ * @param file file expected to be in zip
+ * @throws FileNotFoundException file not found or is not a zip file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExistsInZip(Path path, String file) throws IOException
{
+fileSystem.copyToLocalFile(path, new Path("/tmp"));
+File dst = new File((new Path("/tmp", path.getName())).toString());
+Enumeration entries;
+ZipFile zipFile = new ZipFile(dst);
+boolean found = false;
+
+try {
+ entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ZipEntry entry = entries.nextElement();
+String nm = entry.getName();
+if (nm.endsWith(file)) {
+ found = true;
+ break;
+}
+ }
+} finally {
+ zipFile.close();
+}
+dst.delete();
+if (!found) throw new FileNotFoundException("file: " + file + " not found
in " + path);
+ }
+ /**
* Create the application-instance specific temporary directory
* in the DFS
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 3835df6..2ce16c7 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -92,6 +92,12 @@ public class AgentClientProvider extends
AbstractClientProvider
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
Path appDefPath = new Path(appDef);
sliderFileSystem.verifyFileExists(appDefPath);
+
+String pkgList = instanceDefinition.getAppConfOperations().
+getGlobalOptions().getOption(AgentKeys.PACKAGE_LIST, null);
+if (pkgList != null) {
+ sliderFileSystem.verifyFileExistsInZip(appDefPath, pkgList);
+}
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
[03/38] git commit: SLIDER-90: HA NN, RM support
SLIDER-90: HA NN, RM support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b52d4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b52d4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b52d4b0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0b52d4b0e2fc0a3a5e98c34bb6248a84e359c3db
Parents: 7ab9ce9
Author: Steve Loughran
Authored: Tue Aug 5 12:04:41 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 12:04:41 2014 +0100
--
.../java/org/apache/slider/client/SliderClient.java | 10 --
.../slider/common/params/AbstractActionArgs.java | 14 +-
.../java/org/apache/slider/common/params/ArgOps.java | 7 ---
.../org/apache/slider/common/params/ClientArgs.java | 5 +++--
.../org/apache/slider/common/params/CommonArgs.java | 11 +--
.../slider/server/appmaster/SliderAppMaster.java | 4 ++--
6 files changed, 23 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..ff2e865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -182,7 +182,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
serviceArgs.applyDefinitions(conf);
-serviceArgs.applyFileSystemURL(conf);
+serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
SliderUtils.forceLogin();
@@ -330,9 +330,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
-} catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
-} catch (KeeperException e) {
+} catch (InterruptedException | KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1008,8 +1006,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
}
-if (serviceArgs.getFilesystemURL() != null) {
- commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemURL());
+if (serviceArgs.getFilesystemBinding() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM,
serviceArgs.getFilesystemBinding());
}
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 44bc239..3a8cea4 100644
---
a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++
b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -25,7 +25,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -41,26 +40,23 @@ public abstract class AbstractActionArgs extends ArgOps
implements Arguments {
}
/**
- * URI of the filesystem
+ * URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem URI",
- converter = URIArgumentConverter.class)
- public URI filesystemURL;
+ description = "Filesystem Binding")
+ public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
-
/**
* This is the default parameter
*/
@Parameter
public final List parameters = new ArrayList<>();
-
/**
* get the name: relies on arg 1 being the cluster name in all
[28/38] git commit: Merge branch 'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release' into develop
Merge branch 'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d19c8022 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d19c8022 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d19c8022 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: d19c802264afc9121cd9470860d2ac66caf5357f Parents: 32fd3f5 0646c23 Author: Steve Loughran Authored: Wed Aug 6 19:25:17 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 19:25:17 2014 +0100 -- .../apache/slider/common/tools/Comparators.java | 58 + .../providers/AbstractProviderService.java | 29 ++- .../slider/providers/ProviderService.java | 20 +- .../appmaster/AsyncRMOperationHandler.java | 51 .../server/appmaster/SliderAppMaster.java | 19 +- .../operations/AbstractRMOperation.java | 29 +++ .../operations/AsyncRMOperationHandler.java | 51 .../operations/ContainerReleaseOperation.java | 44 .../operations/ContainerRequestOperation.java | 44 .../ProviderNotifyingOperationHandler.java | 43 .../operations/RMOperationHandler.java | 35 +++ .../operations/RMOperationHandlerActions.java | 28 ++ .../appmaster/state/AbstractRMOperation.java| 31 --- .../slider/server/appmaster/state/AppState.java | 135 +++--- .../state/ContainerReleaseOperation.java| 44 .../state/ContainerReleaseSelector.java | 38 +++ .../state/ContainerRequestOperation.java| 44 .../MostRecentContainerReleaseSelector.java | 52 .../server/appmaster/state/NodeInstance.java| 12 +- .../slider/server/appmaster/state/NodeMap.java | 54 .../appmaster/state/RMOperationHandler.java | 43 .../server/appmaster/state/RoleHistory.java | 14 +- .../appmaster/state/SimpleReleaseSelector.java | 34 +++ .../agent/TestAgentAMManagementWS.groovy| 11 +- .../TestAppStateContainerFailure.groovy | 166 .../appstate/TestAppStateDynamicRoles.groovy| 93 --- .../TestAppStateRebuildOnAMRestart.groovy | 124 - .../appstate/TestAppStateRolePlacement.groovy | 99 --- .../appstate/TestAppStateRoleRelease.groovy | 82 -- .../TestContainerResourceAllocations.groovy | 108 .../model/appstate/TestFlexDynamicRoles.groovy | 189 -- .../TestMockAppStateContainerFailure.groovy | 166 .../TestMockAppStateDynamicRoles.groovy | 94 +++ .../TestMockAppStateFlexDynamicRoles.groovy | 190 ++ .../appstate/TestMockAppStateFlexing.groovy | 116 + .../TestMockAppStateRMOperations.groovy | 180 + .../TestMockAppStateRebuildOnAMRestart.groovy | 125 + .../TestMockAppStateRolePlacement.groovy| 102 .../appstate/TestMockAppStateRoleRelease.groovy | 82 ++ .../TestMockContainerResourceAllocations.groovy | 108 .../model/appstate/TestMockFlexing.groovy | 128 - .../model/appstate/TestMockRMOperations.groovy | 176 - .../history/TestFindNodesForNewInstances.groovy | 128 - .../history/TestFindNodesForRelease.groovy | 146 --- .../model/history/TestHistoryRW.groovy | 258 --- .../model/history/TestHistoryRWOrdering.groovy | 145 --- .../model/history/TestNIComparators.groovy | 77 -- .../TestOutstandingRequestTracker.groovy| 73 -- .../TestRoleHistoryContainerEvents.groovy | 17 +- ...stRoleHistoryFindNodesForNewInstances.groovy | 128 + .../history/TestRoleHistoryNIComparators.groovy | 77 ++ ...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++ .../model/history/TestRoleHistoryRW.groovy | 258 +++ .../history/TestRoleHistoryRWOrdering.groovy| 145 +++ .../model/mock/BaseMockAppStateTest.groovy | 3 +- .../model/mock/MockProviderService.groovy | 69 +++-- .../model/mock/MockRMOperationHandler.groovy| 8 +- .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 3 +- 60 files changed, 2544 insertions(+), 2364 deletions(-) --
[21/38] git commit: SLIDER-295 Unit test failure caused by wrong parameter type in AppState#onCompletedNode()
SLIDER-295 Unit test failure caused by wrong parameter type in
AppState#onCompletedNode()
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/533b229f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/533b229f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/533b229f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 533b229f09d7fd691be53c2edaa364c72896412f
Parents: 32fd3f5
Author: tedyu
Authored: Wed Aug 6 08:23:23 2014 -0700
Committer: tedyu
Committed: Wed Aug 6 08:23:23 2014 -0700
--
.../java/org/apache/slider/server/appmaster/state/AppState.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/533b229f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 08d43db..3acd222 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1218,7 +1218,7 @@ public class AppState {
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
message = String.format("Failure %s on host %s: %s",
-roleInstance.getContainerId(),
+roleInstance.getContainerId().toString(),
failedContainer.getNodeId().getHost(),
completedLogsUrl);
} else {
[01/38] git commit: SLIDER-265 Add role for HBase REST gateway (Sumit)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry f7b24156d ->
04e5b92a7
SLIDER-265 Add role for HBase REST gateway (Sumit)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/94263fc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/94263fc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/94263fc1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 94263fc18e7f6e835071d47f3a40b25f47c7ff88
Parents: 067c306
Author: tedyu
Authored: Mon Aug 4 13:06:10 2014 -0700
Committer: tedyu
Committed: Mon Aug 4 13:06:10 2014 -0700
--
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/metainfo.xml | 18 ++
.../hbase/package/scripts/hbase_regionserver.py | 3 ---
.../hbase/package/scripts/hbase_service.py| 2 ++
app-packages/hbase/package/scripts/params.py | 2 ++
app-packages/hbase/resources.json | 5 +
.../slider/server/appmaster/state/NodeEntry.java | 1 -
7 files changed, 28 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94263fc1/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 03349eb..8b0825d 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -23,6 +23,7 @@
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
+"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94263fc1/app-packages/hbase/metainfo.xml
--
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index 99413f6..c14b06f 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -41,6 +41,10 @@
http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status
+org.apache.slider.hbase.rest
+
http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}
+
+
org.apache.slider.metrics
http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}
@@ -56,6 +60,10 @@
HBASE_REGIONSERVER-START
HBASE_MASTER-STARTED
+
+HBASE_MASTER-START
+HBASE_REST-INSTALLED
+
@@ -81,6 +89,16 @@
+HBASE_REST
+SLAVE
+1
+
+ scripts/hbase_rest.py
+ PYTHON
+
+
+
+
HBASE_CLIENT
CLIENT
0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94263fc1/app-packages/hbase/package/scripts/hbase_regionserver.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_regionserver.py
b/app-packages/hbase/package/scripts/hbase_regionserver.py
index 8d66dcc..daa5732 100644
--- a/app-packages/hbase/package/scripts/hbase_regionserver.py
+++ b/app-packages/hbase/package/scripts/hbase_regionserver.py
@@ -58,9 +58,6 @@ class HbaseRegionServer(Script):
pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
check_process_status(pid_file)
- def decommission(self, env):
-print "Decommission not yet implemented!"
-
if __name__ == "__main__":
HbaseRegionServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94263fc1/app-packages/hbase/package/scripts/hbase_service.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_service.py
b/app-packages/hbase/package/scripts/hbase_service.py
index 2b30083..cb9c599 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -35,6 +35,8 @@ def hbase_service(
if action == 'start':
daemon_cmd = format("{cmd} start {role}")
+ if name == 'rest':
+daemon_cmd = format("{daemon_cmd} -p {rest_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat
{pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd}
[10/38] git commit: SLIDER-270 more work on the double flex test case
SLIDER-270 more work on the double flex test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/291a82ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/291a82ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/291a82ef
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 291a82ef1e99a6590e245ebce7c0d1d861eac263
Parents: 040ef79
Author: Steve Loughran
Authored: Tue Aug 5 21:09:00 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:09:00 2014 +0100
--
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ---
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 1 +
.../minicluster/flexing/TestClusterFlexDownMultiple.groovy| 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 49dc02e..ed66190 100644
---
a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -136,6 +136,9 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
}
}
+ protected String buildClustername(String clustername) {
+return clustername ?: createClusterName()
+ }
/**
* Create the cluster name from the method name and an auto-incrementing
@@ -240,9 +243,7 @@ public abstract class YarnMiniClusterTestBase extends
ServiceLauncherBaseTest {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
-if (!name) {
- name = createClusterName()
-}
+name = buildClustername(name)
miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs,
numLogDirs)
miniCluster.init(conf)
miniCluster.start();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index c9f0a4f..6a69e17 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -284,6 +284,7 @@ public abstract class HBaseMiniClusterTestBase extends
YarnZKMiniClusterTestBase
int workers,
int workerFlexTarget,
boolean testHBaseAfter) {
+clustername = buildClustername(clustername);
SliderClient sliderClient = startHBaseCluster(clustername, masters,
workers)
//now flex
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
index 43ade73..97a9b35 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -34,13 +34,13 @@ class TestClusterFlexDownMultiple extends
HBaseMiniClusterTestBase {
@Test
public void testClusterFlexDownMultiple() throws Throwable {
def clusterName = createClusterName();
-SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+SliderClient sliderClient = startHBaseCluster(clus
[08/38] git commit: SLIDER-270 improving diags and failure log generation in AppState release operations
SLIDER-270 improving diags and failure log generation in AppState release
operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/268253c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/268253c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/268253c1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 268253c1bb1582d4d1879c4798064e59bb447da0
Parents: ae4d073
Author: Steve Loughran
Authored: Tue Aug 5 21:07:23 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:08:05 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 69 +---
.../server/appmaster/state/RoleStatus.java | 4 +-
2 files changed, 46 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f8a39d..74e91a0 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -696,7 +696,7 @@ public class AppState {
roleStatusMap.get(priority));
}
roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+new RoleStatus(providerRole));
roles.put(providerRole.name, providerRole);
}
@@ -1175,10 +1175,14 @@ public class AppState {
log.info("Container was queued for release");
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- log.info("decrementing role count for role {}", roleStatus.getName());
- roleStatus.decReleasing();
- roleStatus.decActual();
- roleStatus.incCompleted();
+ int releasing = roleStatus.decReleasing();
+ int actual = roleStatus.decActual();
+ int completedCount = roleStatus.incCompleted();
+ log.info("decrementing role count for role {} to {}; releasing={},
completed={}",
+ roleStatus.getName(),
+ actual,
+ releasing,
+ completedCount);
roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
@@ -1205,25 +1209,17 @@ public class AppState {
roleStatus.decActual();
boolean shortLived = isShortLived(roleInstance);
String message;
- if (roleInstance.container != null) {
-String user = null;
-try {
- user = SliderUtils.getCurrentUser().getShortUserName();
-} catch (IOException ignored) {
-}
-String completedLogsUrl = null;
-Container c = roleInstance.container;
-String url = logServerURL;
-if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + roleInstance.getContainerId()
+ "/ctx/" + user;
-}
-message = String.format("Failure %s on host %s" +
-(completedLogsUrl != null ? ", see %s" : ""),
roleInstance.getContainerId(),
-c.getNodeId().getHost(), completedLogsUrl);
+ Container failedContainer = roleInstance.container;
+
+ //build the failure message
+ if (failedContainer != null) {
+String completedLogsUrl = getLogsURLForContainer(failedContainer);
+message = String.format("Failure %s on host %s: %s" +
+roleInstance.getContainerId(),
+failedContainer.getNodeId().getHost(),
+completedLogsUrl);
} else {
-message = String.format("Failure %s",
-containerId.toString());
+message = String.format("Failure %s", containerId);
}
roleStatus.noteFailed(message);
//have a look to see if it short lived
@@ -1231,8 +1227,8 @@ public class AppState {
roleStatus.incStartFailed();
}
- if (roleInstance.container != null) {
-roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ if (failedContainer!= null) {
+roleHistory.onFailedContainer(failedContainer, shortLived);
}
} catch (YarnRuntimeException e1) {
@@ -1270,6 +1266,29 @@ public class AppState {
return result;
}
+ /**
+ * Get the URL log for
[26/38] git commit: IDE-indicated import cleanup
IDE-indicated import cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/868f7e2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/868f7e2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/868f7e2c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 868f7e2c6319040d4f9edea202672f18b7939f1a
Parents: 3bb202f
Author: Steve Loughran
Authored: Wed Aug 6 17:29:18 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:29:18 2014 +0100
--
.../providers/agent/TestAgentAMManagementWS.groovy | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/868f7e2c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index d3a0be7..e248ec3 100644
---
a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -27,7 +27,6 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
import org.apache.slider.server.services.security.CertificateManager
@@ -37,13 +36,15 @@ import org.junit.Test
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import javax.net.ssl.HostnameVerifier
+import javax.net.ssl.HttpsURLConnection
+import javax.net.ssl.SSLSession
import javax.ws.rs.core.MediaType
import static org.apache.slider.common.params.Arguments.ARG_OPTION
import static org.apache.slider.providers.agent.AgentKeys.*
import static
org.apache.slider.providers.agent.AgentTestUtils.createDummyJSONRegister
import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
-import static org.apache.slider.test.SliderTestUtils.log
@CompileStatic
@Slf4j
@@ -53,10 +54,10 @@ class TestAgentAMManagementWS extends AgentTestBase {
final static Logger logger =
LoggerFactory.getLogger(TestAgentAMManagementWS.class)
static {
//for localhost testing only
-javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
-new javax.net.ssl.HostnameVerifier(){
+HttpsURLConnection.setDefaultHostnameVerifier(
+new HostnameVerifier(){
public boolean verify(String hostname,
- javax.net.ssl.SSLSession sslSession)
{
+ SSLSession sslSession) {
logger.info("verifying hostname ${hostname}")
InetAddress[] addresses =
InetAddress.getAllByName(hostname);
[05/38] git commit: SLIDER-283 Add role for HBase THRIFT gateway
SLIDER-283 Add role for HBase THRIFT gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9a91b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9a91b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9a91b46
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a9a91b46662ecf969cbc7c5e69cef214fc91e086
Parents: 0b52d4b
Author: tedyu
Authored: Tue Aug 5 09:32:53 2014 -0700
Committer: tedyu
Committed: Tue Aug 5 09:32:53 2014 -0700
--
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/metainfo.xml | 15 +
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift.py | 62
app-packages/hbase/package/scripts/params.py| 1 +
app-packages/hbase/resources.json | 5 ++
.../providers/hbase/HBaseClientProvider.java| 2 +-
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 3 +
10 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 8b0825d..f0ab144 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -23,7 +23,7 @@
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
-"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/metainfo.xml
--
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index c14b06f..ee6ddea 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}
+org.apache.slider.hbase.thrift
+
http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}
+
+
org.apache.slider.metrics
http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}
@@ -63,6 +67,7 @@
HBASE_MASTER-START
HBASE_REST-INSTALLED
+HBASE_THRIFT-INSTALLED
@@ -99,6 +104,16 @@
+HBASE_THRIFT
+SLAVE
+1
+
+ scripts/hbase_thrift.py
+ PYTHON
+
+
+
+
HBASE_CLIENT
CLIENT
0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_service.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_service.py
b/app-packages/hbase/package/scripts/hbase_service.py
index cb9c599..01d38b3 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -37,6 +37,8 @@ def hbase_service(
daemon_cmd = format("{cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
+ elif name == 'thrift':
+daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat
{pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_thrift.py
--
diff --git a/app-packages/hbase/package/scripts/hbase_thrift.py
b/app-packages/hbase/package/scripts/hbase_thrift.py
new file mode 100644
index 000..84bfc62
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. T
[34/38] git commit: SLIDER-286 AM restart count is now set to 0 in AppMasterLauncher/Slider client, so AMs get default cluster value. The value is logged at debug level
SLIDER-286 AM restart count is now set to 0 in AppMasterLauncher/Slider client,
so AMs get default cluster value. The value is logged at debug level
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1350491f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1350491f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1350491f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1350491fd487987b458da75dc571c07c1ae76770
Parents: 56083ec
Author: Steve Loughran
Authored: Thu Aug 7 17:36:45 2014 +0100
Committer: Steve Loughran
Committed: Thu Aug 7 17:36:45 2014 +0100
--
.../src/main/java/org/apache/slider/client/SliderClient.java | 4 ++--
.../java/org/apache/slider/core/launch/AppMasterLauncher.java | 7 +--
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ff2e865..e1d083c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -833,8 +833,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
-amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT,
- DEFAULT_AM_RESTART_LIMIT));
+int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
+amLauncher.setMaxAppAttempts(maxAppAttempts);
sliderFileSystem.purgeAppInstanceTempFiles(clustername);
Path tempPath = sliderFileSystem.createAppInstanceTempPath(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index dc4a886..bd8a0a5 100644
---
a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++
b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -50,7 +50,7 @@ public class AppMasterLauncher extends AbstractLauncher {
private final ApplicationSubmissionContext submissionContext;
private final ApplicationId appId;
private final boolean secureCluster;
- private int maxAppAttempts = 2;
+ private int maxAppAttempts = 0;
private boolean keepContainersOverRestarts = true;
private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private int priority = 1;
@@ -174,7 +174,10 @@ public class AppMasterLauncher extends AbstractLauncher {
submissionContext.setKeepContainersAcrossApplicationAttempts(true);
}
-submissionContext.setMaxAppAttempts(maxAppAttempts);
+if (maxAppAttempts > 0) {
+ log.debug("Setting max AM attempts to {}", maxAppAttempts);
+ submissionContext.setMaxAppAttempts(maxAppAttempts);
+}
if (secureCluster) {
addSecurityTokens();
[38/38] git commit: SLIDER-149 merge with trunk
SLIDER-149 merge with trunk Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/04e5b92a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/04e5b92a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/04e5b92a Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 04e5b92a737a4dde7866e99fd2b1947519f51fbc Parents: f7b2415 66cbd37 Author: Steve Loughran Authored: Thu Aug 7 21:03:52 2014 +0100 Committer: Steve Loughran Committed: Thu Aug 7 21:03:52 2014 +0100 -- app-packages/app-pkg-template/README.txt| 34 ++ app-packages/app-pkg-template/appConfig.json| 21 + app-packages/app-pkg-template/metainfo.xml | 57 ++ .../package/files/myapp-1.0.0.tar.REPLACE | 16 + .../package/scripts/myapp_master.py | 46 ++ .../app-pkg-template/package/scripts/params.py | 30 ++ app-packages/app-pkg-template/resources.json| 16 + app-packages/hbase/appConfig.json | 1 + app-packages/hbase/metainfo.xml | 33 ++ .../hbase/package/scripts/hbase_regionserver.py | 3 - .../hbase/package/scripts/hbase_rest.py | 62 +++ .../hbase/package/scripts/hbase_service.py | 4 + .../hbase/package/scripts/hbase_thrift.py | 62 +++ app-packages/hbase/package/scripts/params.py| 3 + app-packages/hbase/resources.json | 10 + app-packages/memcached-win/appConfig.json | 6 +- app-packages/memcached-win/metainfo.xml | 8 +- .../memcached-win/package/scripts/memcached.py | 2 +- .../memcached-win/package/scripts/params.py | 1 + app-packages/memcached/appConfig.json | 7 +- app-packages/memcached/metainfo.xml | 8 +- .../memcached/package/scripts/memcached.py | 2 +- .../memcached/package/scripts/params.py | 1 + pom.xml | 2 +- .../python/agent/CustomServiceOrchestrator.py | 86 ++- .../core/providers/windows/system.py| 54 +- .../agent/TestCustomServiceOrchestrator.py | 127 - slider-agent/src/test/python/agent/TestGrep.py | 2 +- slider-agent/src/test/python/agent/TestMain.py | 8 +- .../src/test/python/agent/TestRegistration.py | 4 +- .../org/apache/slider/client/SliderClient.java | 14 +- .../common/params/AbstractActionArgs.java | 14 +- .../org/apache/slider/common/params/ArgOps.java | 7 +- .../apache/slider/common/params/ClientArgs.java | 5 +- .../apache/slider/common/params/CommonArgs.java | 11 +- .../apache/slider/common/tools/Comparators.java | 58 ++ .../slider/common/tools/CoreFileSystem.java | 33 ++ .../slider/core/launch/AppMasterLauncher.java | 7 +- .../providers/AbstractProviderService.java | 41 +- .../slider/providers/ProviderService.java | 39 +- .../providers/agent/AgentClientProvider.java| 6 + .../providers/agent/AgentProviderService.java | 534 --- .../providers/agent/ComponentCommandOrder.java | 8 +- .../providers/agent/ComponentInstanceState.java | 80 ++- .../providers/agent/HeartbeatMonitor.java | 66 ++- .../agent/application/metadata/Application.java | 9 + .../agent/application/metadata/Component.java | 13 + .../application/metadata/MetainfoParser.java| 5 + .../server/appmaster/AMViewForProviders.java| 21 +- .../appmaster/AsyncRMOperationHandler.java | 51 -- .../server/appmaster/SliderAppMaster.java | 95 ++-- .../operations/AbstractRMOperation.java | 29 + .../operations/AsyncRMOperationHandler.java | 51 ++ .../operations/ContainerReleaseOperation.java | 44 ++ .../operations/ContainerRequestOperation.java | 44 ++ .../ProviderNotifyingOperationHandler.java | 43 ++ .../operations/RMOperationHandler.java | 35 ++ .../operations/RMOperationHandlerActions.java | 28 + .../appmaster/state/AbstractRMOperation.java| 31 -- .../slider/server/appmaster/state/AppState.java | 275 ++ .../state/ContainerReleaseOperation.java| 44 -- .../state/ContainerReleaseSelector.java | 38 ++ .../state/ContainerRequestOperation.java| 44 -- .../MostRecentContainerReleaseSelector.java | 52 ++ .../server/appmaster/state/NodeEntry.java | 1 - .../server/appmaster/state/NodeInstance.java| 12 +- .../slider/server/appmaster/state/NodeMap.java | 54 -- .../appmaster/state/RMOperationHandler.java | 43 -- .../server/appmaster/state/RoleHistory.java | 20 +- .../server/appmaster/state/RoleStatus.java | 4 +- .../appmaster/state/SimpleReleaseSelector.java | 34 ++ .../agent/AgentMiniClusterTestBase.groovy | 7 +- .../agent/actions/TestActionExists.groovy | 7 +- .../slider/agent/actions/TestActionList.groovy | 5 +- .../age
[20/38] git commit: SLIDER-276 : AgentProvider releases nodes that the AM has already been released after detecting heartbeat failure
SLIDER-276 : AgentProvider releases nodes that the AM has already been released
after detecting heartbeat failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/32fd3f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/32fd3f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/32fd3f56
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 32fd3f56a785e29b55c0560c6256e50bf07215b8
Parents: d68be5a
Author: Steve Loughran
Authored: Wed Aug 6 14:41:00 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 14:41:00 2014 +0100
--
.../providers/agent/AgentProviderService.java | 34 ++---
.../providers/agent/ComponentCommandOrder.java | 8 +-
.../providers/agent/ComponentInstanceState.java | 80 +++---
.../providers/agent/HeartbeatMonitor.java | 66 +---
.../server/appmaster/AMViewForProviders.java| 21 ++-
.../server/appmaster/SliderAppMaster.java | 27 ++--
.../slider/server/appmaster/state/AppState.java | 6 +-
.../appmaster/model/mock/MockContainer.groovy | 2 +-
.../agent/TestAgentProviderService.java | 151 +++
.../agent/TestComponentCommandOrder.java| 17 ++-
.../agent/TestComponentInstanceState.java | 124 ---
.../providers/agent/TestHeartbeatMonitor.java | 120 ++-
12 files changed, 337 insertions(+), 319 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 6cfadf3..4d54509 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -58,6 +58,7 @@ import
org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -127,10 +128,10 @@ public class AgentProviderService extends
AbstractProviderService implements
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
- private Map componentStatuses = new
ConcurrentHashMap<>();
- private Map> componentInstanceData = new
ConcurrentHashMap();
+ private final Map componentStatuses = new
ConcurrentHashMap<>();
+ private final Map> componentInstanceData = new
ConcurrentHashMap<>();
private final Map> allocatedPorts = new
ConcurrentHashMap<>();
- private Map workFolders =
+ private final Map workFolders =
Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
@@ -291,7 +292,7 @@ public class AgentProviderService extends
AbstractProviderService implements
getComponentStatuses().put(label,
new ComponentInstanceState(
role,
- container.getId().toString(),
+ container.getId(),
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
@@ -333,7 +334,7 @@ public class AgentProviderService extends
AbstractProviderService implements
String label = registration.getHostname();
if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
-
getComponentStatuses().get(label).setLastHeartbeat(System.currentTimeMillis());
+ getComponentStatuses().get(label).heartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
@@ -371,7 +372,7 @@ public class AgentProviderService extends
AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
-componentStatus.setLastHeartbeat(System.currentTimeMillis
[14/38] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad2c798d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad2c798d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad2c798d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ad2c798d309aded2ef18dea7aa1f90acdd367b14
Parents: 1894a0c
Author: Sumit Mohanty
Authored: Wed Jul 30 23:44:22 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
.../memcached/package/scripts/memcached.py | 2 +-
.../memcached/package/scripts/params.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 2 +-
.../agent/TestCustomServiceOrchestrator.py | 6 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../providers/agent/AgentProviderService.java | 70 ++--
.../agent/TestAgentProviderService.java | 4 +-
7 files changed, 76 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/memcached.py
--
diff --git a/app-packages/memcached/package/scripts/memcached.py
b/app-packages/memcached/package/scripts/memcached.py
index b2df89a..cf04cf1 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -33,7 +33,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
-process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val}")
+process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val}
-classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main
--memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/params.py
--
diff --git a/app-packages/memcached/package/scripts/params.py
b/app-packages/memcached/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp =
config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 7d740c2..f6cdfb2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -250,7 +250,7 @@ class CustomServiceOrchestrator():
self.config.getLogPath())
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
-allocated_ports[k] = value
+allocated_ports[key + "." + k] = value
command['configurations'][key][k] = value
pass
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
--
diff --git
a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3b41bd7..3343a8b 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -226,7 +226,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 0)
-self.assertEqual(ret['allocated_ports'], {'a.port': '10233'})
+self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 1)
@@ -480,8 +480,8 @@ class TestCustomServiceOrchestrator(Te
[18/38] git commit: SLIDER-270: one-liner fix to actually resolve the problem
SLIDER-270: one-liner fix to actually resolve the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d6c3fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d6c3fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d6c3fad
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8d6c3fad0e1edc9c482f43305f82c976e3d53aa9
Parents: df963f2
Author: Steve Loughran
Authored: Wed Aug 6 09:32:36 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 09:33:07 2014 +0100
--
.../apache/slider/server/appmaster/state/AppState.java | 13 ++---
.../server/appmaster/model/mock/MockYarnEngine.groovy | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b17c3a..db74ea8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -786,7 +786,10 @@ public class AppState {
public RoleInstance getActiveContainer(ContainerId id) {
return activeContainers.get(id);
}
-
+
+ private RoleInstance removeActiveContainer(ContainerId id) {
+return activeContainers.remove(id);
+ }
public synchronized List cloneLiveContainerInfoList() {
List allRoleInstances;
@@ -1265,11 +1268,15 @@ public class AppState {
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
+
+// and the active node list if present
+removeActiveContainer(containerId);
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
-containerId) : "container in release queue";
+containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
-containerId) : " container in live nodes";
+containerId) : " container still in live nodes";
assert getActiveContainer(containerId) ==
null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 763f653..b57a79a 100644
---
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -104,7 +104,7 @@ class MockYarnEngine {
if (op instanceof ContainerReleaseOperation) {
ContainerReleaseOperation cro = (ContainerReleaseOperation) op
ContainerId cid = cro.containerId
-releaseContainer(cid);
+assert releaseContainer(cid);
released.add(cid)
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
[11/38] git commit: SLIDER-270 assertion checking has found the location of the problem
SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f
Parents: 291a82e
Author: Steve Loughran
Authored: Tue Aug 5 21:35:28 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 21:35:54 2014 +0100
--
.../slider/server/appmaster/state/AppState.java | 25 ++--
.../server/appmaster/state/RoleHistory.java | 6 ++---
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../src/test/resources/log4j.properties | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 74e91a0..2b17c3a 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1172,7 +1172,7 @@ public class AppState {
RoleInstance roleInstance;
if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release");
+ log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
int releasing = roleStatus.decReleasing();
@@ -1183,7 +1183,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container);
+ roleHistory.onReleaseCompleted(container, true);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1251,18 +1251,28 @@ public class AppState {
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
+log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
-if (node == null) {
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
-
-} else {
+if (node != null) {
node.state = ClusterDescription.STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
result.roleInstance = node;
+} else {
+ // not in the list
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
}
+// finally, verify the node doesn't exist any more
+assert !containersBeingReleased.containsKey(
+containerId) : "container in release queue";
+assert !getLiveNodes().containsKey(
+containerId) : " container in live nodes";
+assert getActiveContainer(containerId) ==
+ null : "Container still in active container list";
+
return result;
}
@@ -1422,6 +1432,7 @@ public class AppState {
* @throws SliderInternalStateException if the operation reveals that
* the internal state of the application is inconsistent.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public List reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List operations = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0cd2b39..de7e9d8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHead
[30/38] git commit: SLIDER-298. Add an application package template
SLIDER-298. Add an application package template
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/609fcd32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/609fcd32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/609fcd32
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 609fcd327c4ee814e4f33b0357bdef9e77caf0fc
Parents: ec76068
Author: Sumit Mohanty
Authored: Wed Aug 6 13:55:14 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Aug 6 13:55:14 2014 -0700
--
app-packages/app-pkg-template/README.txt| 34 +++
app-packages/app-pkg-template/appConfig.json| 21 +++
app-packages/app-pkg-template/metainfo.xml | 57 +++
app-packages/app-pkg-template/myapp-1.0.0.zip | Bin 0 -> 5455 bytes
.../package/files/myapp-1.0.0.tar.REPLACE | 16 ++
.../package/scripts/myapp_master.py | 46 +++
.../app-pkg-template/package/scripts/params.py | 30 ++
app-packages/app-pkg-template/resources.json| 16 ++
8 files changed, 220 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/README.txt
--
diff --git a/app-packages/app-pkg-template/README.txt
b/app-packages/app-pkg-template/README.txt
new file mode 100644
index 000..00dfdbc
--- /dev/null
+++ b/app-packages/app-pkg-template/README.txt
@@ -0,0 +1,34 @@
+
+
+How to create a Slider app package for myapp?
+
+To create the app package you will need the application tarball copied to a
specific location.
+
+E.g.
+ cp ~/Downloads/myapp-1.0.0.tar package/files/
+
+Create a zip package at the root of the package (/app-packages/myapp/)
+ zip -r myapp-1.0.0.zip .
+
+Verify the content using
+ zip -Tv myapp-1.0.0.zip
+
+While appConfig.json and resources.json are not required for the package they
work
+well as the default configuration for Slider apps. So its advisable that when
you
+create an application package for Slider, include sample/default
resources.json and
+appConfig.json for a one-node Yarn cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/appConfig.json
--
diff --git a/app-packages/app-pkg-template/appConfig.json
b/app-packages/app-pkg-template/appConfig.json
new file mode 100644
index 000..a6f61f9
--- /dev/null
+++ b/app-packages/app-pkg-template/appConfig.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0";,
+ "metadata": {
+ },
+ "global": {
+"application.def": "package/myapp-1.0.0.zip",
+"java_home": "/usr/jdk64/jdk1.7.0_45",
+
+"site.global.app_user": "yarn",
+"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
+
+"site.global.listen_port": "${MYAPP_COMPONENT.ALLOCATED_PORT}"
+ },
+ "components": {
+"slider-appmaster": {
+ "jvm.heapsize": "256M"
+},
+"MYAPP_COMPONENT": {
+}
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/metainfo.xml
--
diff --git a/app-packages/app-pkg-template/metainfo.xml
b/app-packages/app-pkg-template/metainfo.xml
new file mode 100644
index 000..c6e1485
--- /dev/null
+++ b/app-packages/app-pkg-template/metainfo.xml
@@ -0,0 +1,57 @@
+
+
+
+
+ 2.0
+
+MyApp
+MyApp is an app that can run on YARN.
+1.0.0
+None
+
+
+
+MYAPP_COMPONENT
+MASTER
+
+
+host_port
+${THIS_HOST}:${site.global.listen_port}
+
+
+
+ scripts/myapp_component.py
+ PYTHON
+
+
+
+
+
+
+any
+
+
+tarball
+files/myapp-1.0.0.tar
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/myapp-1.0.0.zip
--
diff --git a/app-packages/app-pkg-template/myapp-1.0.0.zip
b/app-packages/app-pkg-template/myapp-1.0.0.zip
new file mode 100644
index 000..80dbbb7
Binary files /dev/null and b/app-packages/app-pkg-template/myapp-1.0.0.zip
differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE
--
diff --git
a/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE
b/app-packages/app-pkg-tem
[04/38] git commit: SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock
test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29
Parents: 0b52d4b
Author: Steve Loughran
Authored: Tue Aug 5 16:05:03 2014 +0100
Committer: Steve Loughran
Committed: Tue Aug 5 16:05:03 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 16 +--
.../slider/server/appmaster/state/AppState.java | 43 +++
.../appstate/TestAppStateDynamicRoles.groovy| 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++
.../model/appstate/TestMockRMOperations.groovy | 37 --
.../model/mock/BaseMockAppStateTest.groovy | 57 -
.../appmaster/model/mock/MockYarnEngine.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
11 files changed, 219 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..1ad7fa7 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -671,12 +671,13 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
//build the instance
appState.buildInstance(instanceDefinition,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation);
+ serviceConf,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
// add the AM to the list of nodes in the cluster
@@ -1055,8 +1056,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// non complete containers should not be here
assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- getConfig(), status);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(status);
if (result.containerFailed) {
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index cc238ff..9f8a39d 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.providers.ProviderRole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,6 +249,8 @@ public class AppState {
private long startTimeThreshold;
private int failureThreshold = 10;
+
+ private String logServerURL = "";
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -432,6 +433,7 @@ public class AppState {
/**
* Build up the application state
* @param instanceDefinition definition of the applicatin instance
+ * @param appmasterConfig
* @param publishedProviderConf any configuration info to be published by a
provider
* @param providerRole
[13/38] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dca4818c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dca4818c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dca4818c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: dca4818c551c8fff655c8898c0786aae2fb0fb6a
Parents: ad2c798
Author: Sumit Mohanty
Authored: Fri Aug 1 11:52:35 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
.../slider/providers/ProviderService.java | 7 +
.../providers/agent/AgentProviderService.java | 329 ++-
.../server/appmaster/SliderAppMaster.java | 2 +
.../agent/TestAgentProviderService.java | 49 ++-
4 files changed, 224 insertions(+), 163 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 56e24e9..b9fa34c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -69,6 +70,12 @@ public interface ProviderService extends ProviderCore,
Service,
SliderException;
/**
+ * Notify the providers of container completion
+ * @param containerId
+ */
+ void notifyContainerCompleted(ContainerId containerId);
+
+ /**
* Execute a process in the AM
* @param instanceDefinition cluster description
* @param confDir configuration directory
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b39601a..bbe2001 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
@@ -110,6 +111,7 @@ public class AgentProviderService extends
AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
+ private static final String SHARED_PORT_TAG = "{SHARED}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -290,116 +292,6 @@ public class AgentProviderService extends
AbstractProviderService implements
}
/**
- * Reads and sets the heartbeat monitoring interval. If bad value is
provided then log it and set to default.
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf
instanceDefinition) {
-String hbMonitorInterval = instanceDefinition.getAppConfOperations().
-getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
-
Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
-try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
-}catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
-
[27/38] git commit: SLIDER-296 providers now get notified of container create/release requests
SLIDER-296 providers now get notified of container create/release requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0646c234
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0646c234
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0646c234
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0646c234ff20a98c4620f2dbd69a4a36414ca27d
Parents: 868f7e2
Author: Steve Loughran
Authored: Wed Aug 6 19:24:55 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 19:24:55 2014 +0100
--
.../providers/AbstractProviderService.java | 11 +
.../slider/providers/ProviderService.java | 8 +--
.../appmaster/AsyncRMOperationHandler.java | 51
.../server/appmaster/SliderAppMaster.java | 15 --
.../operations/AbstractRMOperation.java | 29 +++
.../operations/AsyncRMOperationHandler.java | 51
.../operations/ContainerReleaseOperation.java | 44 +
.../operations/ContainerRequestOperation.java | 44 +
.../ProviderNotifyingOperationHandler.java | 43 +
.../operations/RMOperationHandler.java | 35 ++
.../operations/RMOperationHandlerActions.java | 28 +++
.../appmaster/state/AbstractRMOperation.java| 31
.../slider/server/appmaster/state/AppState.java | 5 +-
.../state/ContainerReleaseOperation.java| 44 -
.../state/ContainerRequestOperation.java| 44 -
.../appmaster/state/RMOperationHandler.java | 43 -
.../TestMockAppStateDynamicRoles.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 12 +
.../TestMockAppStateRMOperations.groovy | 4 ++
.../TestMockAppStateRebuildOnAMRestart.groovy | 1 +
.../TestMockAppStateRolePlacement.groovy| 3 ++
.../appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../TestMockContainerResourceAllocations.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 1 +
.../model/mock/MockProviderService.groovy | 11 +
.../model/mock/MockRMOperationHandler.groovy| 8 +--
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +--
27 files changed, 337 insertions(+), 243 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7483a8d..158640d 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -337,4 +338,14 @@ public abstract class AbstractProviderService
public ContainerReleaseSelector createContainerReleaseSelector() {
return new MostRecentContainerReleaseSelector();
}
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+// no-op
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+// no-op
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4278b09..c0a75c1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -34,18 +34,20 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
+import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
[23/38] git commit: SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to
release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ce953ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ce953ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ce953ca
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 5ce953ca301a90e27b1cbe193a482a9cb83c3084
Parents: 32fd3f5
Author: Steve Loughran
Authored: Wed Aug 6 16:58:27 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 16:58:27 2014 +0100
--
.../apache/slider/common/tools/Comparators.java | 58 ++
.../providers/AbstractProviderService.java | 18 +-
.../slider/providers/ProviderService.java | 12 +-
.../server/appmaster/SliderAppMaster.java | 4 +-
.../slider/server/appmaster/state/AppState.java | 134 +
.../state/ContainerReleaseSelector.java | 38
.../MostRecentContainerReleaseSelector.java | 52 +
.../appmaster/state/SimpleReleaseSelector.java | 34
.../TestAppStateContainerFailure.groovy | 166
.../appstate/TestAppStateDynamicRoles.groovy| 93 -
.../TestAppStateRebuildOnAMRestart.groovy | 124
.../appstate/TestAppStateRolePlacement.groovy | 99 --
.../appstate/TestAppStateRoleRelease.groovy | 82
.../TestContainerResourceAllocations.groovy | 108 ---
.../model/appstate/TestFlexDynamicRoles.groovy | 189 --
.../TestMockAppStateContainerFailure.groovy | 166
.../TestMockAppStateDynamicRoles.groovy | 94 +
.../TestMockAppStateFlexDynamicRoles.groovy | 190 +++
.../appstate/TestMockAppStateFlexing.groovy | 126
.../TestMockAppStateRMOperations.groovy | 176 +
.../TestMockAppStateRebuildOnAMRestart.groovy | 124
.../TestMockAppStateRolePlacement.groovy| 99 ++
.../appstate/TestMockAppStateRoleRelease.groovy | 82
.../TestMockContainerResourceAllocations.groovy | 108 +++
.../model/appstate/TestMockFlexing.groovy | 128 -
.../model/appstate/TestMockRMOperations.groovy | 176 -
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 56 +++---
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 3 +-
30 files changed, 1510 insertions(+), 1234 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 000..0ccca0f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+public class Comparators {
+
+ public static class LongComparator implements Comparator, Serializable
{
+@Override
+public int compare(Long o1, Long o2) {
+ long result = o1 - o2;
+ // need to comparisons with a diff greater than integer size
+ if (result < 0 ) {
+return -1;
+ } else if (result >0) {
+return 1;
+ }
+ return 0;
+}
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param the type that is being compared
+ */
+ public static class ComparatorReverser implements
Comparator,
+ Serializable {
+
+final Comparator instance;
+
+public ComparatorReverser(Comparator instance) {
+ this.instance =
[19/38] git commit: Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop
Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d68be5aa Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d68be5aa Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d68be5aa Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: d68be5aa5eaa8c2e5a2d71097eb2907968c512bd Parents: 405aa51 8d6c3fa Author: Steve Loughran Authored: Wed Aug 6 09:33:34 2014 +0100 Committer: Steve Loughran Committed: Wed Aug 6 09:33:34 2014 +0100 -- pom.xml | 2 +- .../server/appmaster/SliderAppMaster.java | 16 +- .../slider/server/appmaster/state/AppState.java | 142 +++--- .../server/appmaster/state/RoleHistory.java | 6 +- .../server/appmaster/state/RoleStatus.java | 4 +- .../appstate/TestAppStateDynamicRoles.groovy| 4 +- .../TestAppStateRebuildOnAMRestart.groovy | 4 +- .../model/appstate/TestFlexDynamicRoles.groovy | 2 + .../model/appstate/TestMockFlexing.groovy | 128 .../model/appstate/TestMockRMOperations.groovy | 37 - .../TestRoleHistoryContainerEvents.groovy | 2 +- .../model/mock/BaseMockAppStateTest.groovy | 57 ++- .../appmaster/model/mock/MockYarnEngine.groovy | 6 +- .../slider/test/YarnMiniClusterTestBase.groovy | 7 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 1 + .../minicluster/HBaseMiniClusterTestBase.groovy | 150 --- .../flexing/TestClusterFlex2To5.groovy | 2 +- .../flexing/TestClusterFlexDownMultiple.groovy | 58 +++ .../TestClusterFlexHMasterFlex1To2.groovy | 45 ++ .../flexing/TestHMasterFlex1To2.groovy | 45 -- .../src/test/resources/log4j.properties | 1 + 22 files changed, 497 insertions(+), 225 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d68be5aa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[15/38] git commit: SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1894a0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1894a0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1894a0ca
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1894a0ca206d4d024f7b3062133ab1039ca5988b
Parents: a9a91b4
Author: Sumit Mohanty
Authored: Tue Jul 29 23:33:00 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached/appConfig.json | 3 +-
app-packages/memcached/metainfo.xml | 6 ++
.../python/agent/CustomServiceOrchestrator.py | 57 --
.../agent/TestCustomServiceOrchestrator.py | 107 +--
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../src/test/python/agent/TestRegistration.py | 4 +-
.../providers/agent/AgentProviderService.java | 3 +
.../agent/application/metadata/Component.java | 13 +++
.../application/metadata/MetainfoParser.java| 4 +
.../agent/TestAgentProviderService.java | 25 -
11 files changed, 202 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index d205591..11fb863 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -10,7 +10,8 @@
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
-"site.global.memory_val": "200M"
+"site.global.memory_val": "200M",
+"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/metainfo.xml
--
diff --git a/app-packages/memcached/metainfo.xml
b/app-packages/memcached/metainfo.xml
index d435b84..0550b7d 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -27,6 +27,12 @@
MEMCACHED
MASTER
+
+
+host_port
+${THIS_HOST}:${site.global.listen_port}
+
+
scripts/memcached.py
PYTHON
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 4eb22bf..7d740c2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -52,9 +52,9 @@ class CustomServiceOrchestrator():
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stdout.txt'))
+
'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stderr.txt'))
+
'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
@@ -81,7 +81,7 @@ class CustomServiceOrchestrator():
script_tuple = (script_path, self.base_dir)
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
-
"structured-out-{0}.json".format(task_id)))
+
"structured-out-{0}.json".format(task_id)))
if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
@@ -97,9 +97,9 @@ class CustomServiceOrchestrator():
for py_file, current_base_dir in filtered_py_file_list:
script_params = [command_name, json_path, current_base_dir]
python_paths =
[os.path.realpath(posixpat
[16/38] git commit: SLIDER-273. Add ability to specify a port as shared across component instances or not
SLIDER-273. Add ability to specify a port as shared across component instances
or not
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/50dc76c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/50dc76c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/50dc76c9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 50dc76c978ef4ff684263b93e8b56619cc947284
Parents: dca4818
Author: Sumit Mohanty
Authored: Sun Aug 3 19:30:13 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached/appConfig.json | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 27 +++-
.../agent/TestCustomServiceOrchestrator.py | 18 ++-
.../providers/AbstractProviderService.java | 5 +
.../providers/agent/AgentProviderService.java | 126 ---
.../model/mock/MockProviderService.groovy | 5 +
.../agent/TestAgentProviderService.java | 58 +++--
.../publisher/TestAgentProviderService.java | 4 +-
8 files changed, 185 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index 11fb863..78cb486 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -11,7 +11,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
-"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
+"site.global.listen_port":
"${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index f6cdfb2..0f4acc6 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -233,12 +233,17 @@ class CustomServiceOrchestrator():
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
+ ALLOCATED_PORT is a hint to allocate port. It works as follows:
+ Its of the form
{component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Either a port gets allocated or if not then just set the value to "0"
"""
def finalize_command(self, command, store_config, allocated_ports):
component = command['componentName']
-allocated_port_format = "${{{0}.ALLOCATED_PORT}}"
-port_allocation_req = allocated_port_format.format(component)
+allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
+allocated_for_any = ".ALLOCATED_PORT}"
+
+port_allocation_req = allocated_for_this_component_format.format(component)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -251,6 +256,10 @@ class CustomServiceOrchestrator():
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
allocated_ports[key + "." + k] = value
+ elif allocated_for_any in value:
+## All unallocated ports should be set to 0
+logger.info("Assigning port 0 " + "for " + value)
+value = "0"
command['configurations'][key][k] = value
pass
pass
@@ -264,8 +273,15 @@ class CustomServiceOrchestrator():
pass
+ """
+ Port allocation can asks for multiple dynamic ports
+ port_req_pattern is of type ${component_name.ALLOCATED_PORT}
+append {DEFAULT_ and find the default value
+append {DO_NOT_PROPAGATE} if it exists
+ """
def allocate_ports(self, value, port_req_pattern):
default_port_pattern = "{DEFAULT_"
+do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
@@ -276,8 +292,15 @@ class CustomServiceOrchestrator():
end_index = value.find("}", start_index)
def_port_str = value[start_index:end_index]
def_port = int(def_port_str)
+# default value of 0 means allocate any dynamic port
+if def_port == 0:
+ def_port = None
+
[24/38] git commit: SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8cecce18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8cecce18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8cecce18
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8cecce1821733c33de3cf9723f9a96aee411305d
Parents: 5ce953c
Author: Steve Loughran
Authored: Wed Aug 6 17:11:39 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:11:39 2014 +0100
--
.../server/appmaster/state/NodeInstance.java| 12 +-
.../server/appmaster/state/RoleHistory.java | 14 +
.../history/TestFindNodesForNewInstances.groovy | 128 -
.../history/TestFindNodesForRelease.groovy | 146 ---
.../model/history/TestHistoryRW.groovy | 258 ---
.../model/history/TestHistoryRWOrdering.groovy | 145 ---
.../model/history/TestNIComparators.groovy | 77 --
.../TestOutstandingRequestTracker.groovy| 73 --
.../TestRoleHistoryContainerEvents.groovy | 17 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 +
.../TestRoleHistoryFindNodesForRelease.groovy | 146 +++
.../history/TestRoleHistoryNIComparators.groovy | 77 ++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++
.../model/history/TestRoleHistoryRW.groovy | 258 +++
.../history/TestRoleHistoryRWOrdering.groovy| 145 +++
15 files changed, 861 insertions(+), 836 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 06375fb..fad60f2 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -75,7 +75,7 @@ public class NodeInstance {
}
/**
- * Cout the number of active role instances on this node
+ * Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
* not being released already.
@@ -84,6 +84,16 @@ public class NodeInstance {
NodeEntry nodeEntry = get(role);
return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
}
+
+ /**
+ * Count the number of live role instances on this node
+ * @param role role index
+ * @return 0 if there are none, otherwise the #of nodes that are running
+ */
+ public int getLiveRoleInstances(int role) {
+NodeEntry nodeEntry = get(role);
+return (nodeEntry != null ) ? nodeEntry.getLive() : 0;
+ }
/**
* Get the entry for a role -and remove it if present
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index de7e9d8..dbb104c 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -529,6 +529,20 @@ public class RoleHistory {
return nodemap.findNodesForRelease(role, count);
}
+
+ /**
+ * Get the list of active nodes ... walks the node map so
+ * is O(nodes)
+ * @param role role index
+ * @return a possibly empty list of nodes with an instance of that node
+ */
+ public synchronized List listActiveNodes(int role) {
+return nodemap.listActiveNodes(role);
+ }
+
+
+
+
/**
* Get the node entry of a container
* @param container container to look up
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
deleted file mode 100644
index d
[22/38] SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
new file mode 100644
index 000..6e18e39
--- /dev/null
+++
b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static
org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static
org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockAppStateRMOperations extends BaseMockAppStateTest implements
MockRoles {
+
+ @Override
+ String getTestName() {
+return "TestMockRMOperations"
+ }
+
+ @Test
+ public void testPriorityOnly() throws Throwable {
+assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTrip() throws Throwable {
+assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTripWithRequest() throws Throwable {
+int priority = buildPriority(5, false)
+assert 5 == extractRole(priority)
+ }
+
+ @Test
+ public void testMockAddOp() throws Throwable {
+role0Status.desired = 1
+List ops = appState.reviewRequestAndReleaseNodes()
+assert ops.size() == 1
+ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+int priority = operation.request.priority.priority
+assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
+RMOperationHandler handler = new MockRMOperationHandler()
+handler.execute(ops)
+
+//tell the container its been allocated
+AbstractRMOperation op = handler.operations[0]
+assert op instanceof ContainerRequestOperation
+ }
+
+ @Test
+ public void testAllocateReleaseOp() throws Throwable {
+role0Status.desired = 1
+
+List ops = appState.reviewRequestAndReleaseNodes()
+ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+AMRMClient.ContainerRequest request = operation.request
+Container cont = engine.allocateContainer(request)
+List allocated = [cont]
+List assignments = [];
+List operations = []
+appState.onContainersAllocated(allocated, assignments, operations)
+assert operations.size() == 0
+assert assignments.size() == 1
+ContainerAssignment assigned = assignments[0]
+Container target = assigned.container
+assert target.id == cont.id
+int roleId = assigned.role.priority
+assert roleId == extractRole(request.priority)
+assert assigned.role.name == ROLE0
+RoleInstance ri = roleInstance(assigned)
+//tell the app it arrived
+appState.containerStartSubmitted(target, ri);
+appState.innerOnNodeManagerContainerStarted(target.id)
+assert role0Status.started == 1
+
+//now release it by changing the role status
+role0Status.desired = 0
+ops = appState.reviewRequestAndReleaseNodes()
+assert ops.size() == 1
+
+assert ops[0] instanceof ContainerReleaseOperation
+ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+assert release.containerId == cont.id
+ }
+
+ @Test
+ public void testComplexAllocation() throws Throwable {
+role0Status.desired = 1
+role1Status.des
[17/38] git commit: SLIDER-272. Application should specify the config types to publish
SLIDER-272. Application should specify the config types to publish
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/640233dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/640233dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/640233dd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 640233dd355f8ac3fe917a307ca3251caf6593e7
Parents: 50dc76c
Author: Sumit Mohanty
Authored: Sun Aug 3 21:48:32 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Aug 5 18:51:58 2014 -0700
--
app-packages/memcached-win/metainfo.xml | 1 +
app-packages/memcached/metainfo.xml | 1 +
.../providers/agent/AgentProviderService.java| 19 +--
.../agent/application/metadata/Application.java | 9 +
.../application/metadata/MetainfoParser.java | 1 +
.../agent/TestAgentProviderService.java | 2 ++
6 files changed, 31 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached-win/metainfo.xml
--
diff --git a/app-packages/memcached-win/metainfo.xml
b/app-packages/memcached-win/metainfo.xml
index a0de8c2..dc2d67b 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -22,6 +22,7 @@
MEMCACHED
Memcache is a network accessible key/value storage system, often
used as a distributed cache.
1.0.0
+None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached/metainfo.xml
--
diff --git a/app-packages/memcached/metainfo.xml
b/app-packages/memcached/metainfo.xml
index 0550b7d..08a504e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -22,6 +22,7 @@
MEMCACHED
Memcache is a network accessible key/value storage system, often
used as a distributed cache.
1.0.0
+None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 28c874b..682efba 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -88,6 +88,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -637,12 +638,26 @@ public class AgentProviderService extends
AbstractProviderService implements
Application application = getMetainfo().getApplication();
List exportGroups = application.getExportGroups();
boolean hasExportGroups = exportGroups != null &&
!exportGroups.isEmpty();
+ Set exportedConfigs = new HashSet();
+ String exportedConfigsStr = application.getExportedConfigs();
+ boolean exportedConfigSpecified = exportedConfigsStr != null;
+ if (application.getExportedConfigs() != null &&
application.getExportedConfigs().length() > 0) {
+for (String exportedConfig :
application.getExportedConfigs().split(",")) {
+ if (exportedConfig.trim().length() > 0) {
+exportedConfigs.add(exportedConfig.trim());
+ }
+}
+ }
+
for (ComponentStatus status : statuses) {
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
-Map configs = status.getConfigs().get(key);
-publishApplicationInstanceData(key, key, configs.entrySet());
+if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
+!exportedConfigSpecified) {
+ Map configs = status.getConfigs().get(key);
+ publishApplicationInstanceData(key, key, configs.entrySet());
+}
}
if (hasExportGroups) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
--
diff --git
a/slider-core/src/main/java
[25/38] git commit: SLIDER-296 removing use of RoleHistory/nodemap target release selection logic - cutting methods out entirely
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic -
cutting methods out entirely
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3bb202f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3bb202f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3bb202f5
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 3bb202f54ce74541a7926b04555732326df7f5d0
Parents: 8cecce1
Author: Steve Loughran
Authored: Wed Aug 6 17:13:40 2014 +0100
Committer: Steve Loughran
Committed: Wed Aug 6 17:13:40 2014 +0100
--
.../slider/server/appmaster/state/NodeMap.java | 54 ---
.../server/appmaster/state/RoleHistory.java | 16 --
.../TestRoleHistoryFindNodesForRelease.groovy | 146 ---
3 files changed, 216 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 32b1656..372a0a5 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -102,60 +102,6 @@ public class NodeMap extends HashMap
{
}
return purged;
}
-
-
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public List findNodesForRelease(int role, int count) {
-List targets = new ArrayList<>(count);
-List active = listActiveNodes(role);
-List multiple = new ArrayList<>();
-int nodesRemaining = count;
-log.debug("searching for {} nodes with candidate set size {}",
- nodesRemaining, active.size());
-ListIterator it = active.listIterator();
-while (it.hasNext() && nodesRemaining > 0) {
- NodeInstance ni = it.next();
- int load = ni.getActiveRoleInstances(role);
- log.debug("Node {} load={}", ni, load);
- assert load != 0;
- if (load == 1) {
-// at the tail of the list, from here active[*] is a load=1 entry
-break;
- }
- // load is >1. Add an entry to the target list FOR EACH INSTANCE ABOVE 1
- for (int i = 0; i < (load - 1) && nodesRemaining > 0; i++) {
-nodesRemaining--;
-log.debug("Push {} #{}", ni, i);
-targets.add(ni);
- }
- // and add to the multiple list
- multiple.add(ni);
- // then pop it from the active list
- it.remove();
-}
-//here either the number is found or there is still some left.
-
-if (nodesRemaining > 0) {
- // leftovers. Append any of the multiple node entries to the tail of
- // the active list (so they get chosen last)
- active.addAll(multiple);
- // all the entries in the list have exactly one node
- // so ask for as many as are needed
- int ask = Math.min(nodesRemaining, active.size());
- log.debug("load=1 nodes to select={} multiples={} available={} ask={}",
-nodesRemaining, multiple.size(),active.size(), ask);
- targets.addAll(active.subList(0, ask));
-}
-return targets;
- }
-
/**
* Clone point
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index dbb104c..e9da081 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -517,19 +517,6 @@ public class RoleHistory {
return requestInstanceOnNode(node, role, resource);
}
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public synchronized List findNodesForRelease(int role, int
count) {
-return nodemap.findNodesForRelease(role, count);
- }
-
-
/**
* Get the list of active nodes ... walks the node map so
* is O(nodes)
@@ -539,9 +526,6
