[08/19] git commit: SLIDER-435. Slider client should accept target yarn queue as an input parameter
SLIDER-435. Slider client should accept target yarn queue as an input parameter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e3c49d59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e3c49d59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e3c49d59
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e3c49d59f6b847b7fc36b7acc9f531782d8336e9
Parents: 0d41051
Author: Sumit Mohanty
Authored: Sun Sep 21 17:23:18 2014 -0700
Committer: Sumit Mohanty
Committed: Sun Sep 21 17:23:18 2014 -0700
--
app-packages/hbase/resources.json | 2 +-
.../org/apache/slider/api/InternalKeys.java | 4 +
.../org/apache/slider/client/SliderClient.java | 7 +-
.../common/SliderXMLConfKeysForTesting.java | 2 +
.../AbstractClusterBuildingActionArgs.java | 4 +
.../apache/slider/common/params/Arguments.java | 1 +
.../slider/core/build/InstanceBuilder.java | 16 +++
.../resources_queue_labels.json | 17 +++
.../providers/agent/TestBuildBasicAgent.groovy | 39 ++-
.../framework/AgentCommandTestBase.groovy | 36 --
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 1 -
.../AppsThroughAgentQueueAndLabelsIT.groovy | 113 +++
.../clusters/remote/slider/slider-client.xml| 14 ++-
13 files changed, 244 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/app-packages/hbase/resources.json
--
diff --git a/app-packages/hbase/resources.json
b/app-packages/hbase/resources.json
index d2fdbd8..c3fec68 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -24,7 +24,7 @@
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
- "yarn.component.instances": "1",
+ "yarn.component.instances": "0",
"yarn.memory": "256"
},
"HBASE_THRIFT2": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ffa3600..5f150e6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -94,6 +94,10 @@ public interface InternalKeys {
*/
String KEYTAB_LOCATION = "internal.keytab.location";
+ /**
+ * Queue used to deploy the app: {@value}
+ */
+ String INTERNAL_QUEUE = "internal.queue";
/**
* Flag to indicate whether or not the chaos monkey is enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3df5b18..221fc43 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -783,6 +783,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
builder.propagateFilename();
builder.propagatePrincipals();
builder.setImageDetailsIfAvailable(buildInfo.getImage(),
buildInfo.getAppHomeDir());
+builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
if (SliderUtils.isUnset(quorum)) {
@@ -1262,7 +1263,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
-
+String suppliedQueue =
internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
+if(!SliderUtils.isUnset(suppliedQueue)) {
+ amQueue = suppliedQueue;
+ log.info("Using queue {} for the application instance.", amQueue);
+}
amLauncher.setQueue(amQueue);
// Submit the application to the applications manager
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
b/slider-core/src/main/java/org/apache/slider/com
[09/19] git commit: SLIDER-434 fix accumulo app package funtests not to reference agent tarball
SLIDER-434 fix accumulo app package funtests not to reference agent tarball
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/00af5a2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/00af5a2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/00af5a2a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 00af5a2a694a3cd115a818e046a223020c4163dc
Parents: e3c49d5
Author: Billie Rinaldi
Authored: Sun Sep 21 19:30:11 2014 -0700
Committer: Billie Rinaldi
Committed: Sun Sep 21 19:30:11 2014 -0700
--
.../org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy | 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/00af5a2a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index f09d3ef..475c4e3 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -129,7 +129,6 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, getClusterName(),
-ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
[19/19] git commit: SLIDER-453 AM queue processing goes live before init complete
SLIDER-453 AM queue processing goes live before init complete
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/94df22c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/94df22c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/94df22c9
Branch: refs/heads/develop
Commit: 94df22c92e48859fbc4b3a560981a6a70cc64b54
Parents: 937e0a7
Author: Steve Loughran
Authored: Tue Sep 23 21:59:01 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 23 21:59:42 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 29 ++--
.../slider/server/appmaster/state/AppState.java | 5 +++-
2 files changed, 30 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 08d0049..365eeab 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -283,6 +283,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
*/
private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
+ /**
+ * Flag set during the init process
+ */
+ private final AtomicBoolean initCompleted = new AtomicBoolean(false);
+
private volatile boolean success = true;
/**
@@ -419,11 +424,18 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+ }
+
+ /**
+ * Start the queue processing
+ */
+ private void startQueueProcessing() {
+log.info("Queue Processing started");
executorService.execute(actionQueues);
executorService.execute(new QueueExecutor(this, actionQueues));
}
- /* === */
+/* === */
/* RunService methods called from ServiceLauncher */
/* === */
@@ -801,7 +813,15 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
+// declare the cluster initialized
+log.info("Application Master Initialization Completed");
+initCompleted.set(true);
+
+// start handling any scheduled events
+
+startQueueProcessing();
// Start the Slider AM provider
+
sliderAMProvider.start();
// launch the real provider; this is expected to trigger a callback that
@@ -1210,8 +1230,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
log.error("Role instance {} failed ", ri);
}
- getProviderService().notifyContainerCompleted(containerId);
- queue(new UnregisterComponentInstance(containerId, 0,
TimeUnit.MILLISECONDS));
+ // known nodes trigger notifications
+ if(!result.unknownNode) {
+getProviderService().notifyContainerCompleted(containerId);
+queue(new UnregisterComponentInstance(containerId, 0,
TimeUnit.MILLISECONDS));
+ }
}
reviewRequestAndReleaseNodes("onContainersCompleted");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 57952e7..abb6fe8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1256,13 +1256,14 @@ public class AppState {
}
/**
- * This is a very small class to send a triple result back from
+ * This is a very small class to send a multiple result back from
* the completion operation
*/
public static class NodeCompletionResult {
public boolean surplusNode = false;
public RoleInstance roleInstance;
public boolean containerFailed;
+public boolean unknownNode = false;
public String toString() {
@@ -1271,6 +1272,7 @@ public class AppState {
sb.append("surplusNode=").append(surplusNode);
sb.append(", roleInsta
[13/19] git commit: SLIDER-440. Add an env.sh file to limit post install changes to conf dir (fix hdp-conf vs. conf)
SLIDER-440. Add an env.sh file to limit post install changes to conf dir (fix hdp-conf vs. conf) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/937e0a7c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/937e0a7c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/937e0a7c Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 937e0a7ce5941b88f024b3e7f6e6503f356274c2 Parents: 39e8365 Author: Sumit Mohanty Authored: Tue Sep 23 10:30:03 2014 -0700 Committer: Sumit Mohanty Committed: Tue Sep 23 10:30:03 2014 -0700 -- slider-assembly/src/conf-hdp/slider-client.xml | 28 - slider-assembly/src/conf-hdp/slider-env.sh | 26 +++ 2 files changed, 42 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/937e0a7c/slider-assembly/src/conf-hdp/slider-client.xml -- diff --git a/slider-assembly/src/conf-hdp/slider-client.xml b/slider-assembly/src/conf-hdp/slider-client.xml index f844106..8e87e53 100644 --- a/slider-assembly/src/conf-hdp/slider-client.xml +++ b/slider-assembly/src/conf-hdp/slider-client.xml @@ -19,42 +19,46 @@ -yarn.log-aggregation-enable -true +slider.zookeeper.quorum +localhost:2181 - slider.yarn.queue default YARN queue for the Application Master - - + + +yarn.log-aggregation-enable +true + + +
[11/19] git commit: SLIDER-180. Changes to allow auto-build of hbase win package (added missing property)
SLIDER-180. Changes to allow auto-build of hbase win package (added missing
property)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7c78605
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7c78605
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7c78605
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e7c78605e768e5a63326c2631e0f86e00368729d
Parents: a1086e0
Author: Sumit Mohanty
Authored: Tue Sep 23 06:58:35 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Sep 23 06:58:35 2014 -0700
--
app-packages/hbase-win/appConfig.json | 1 +
.../org/apache/slider/providers/agent/AgentProviderService.java| 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7c78605/app-packages/hbase-win/appConfig.json
--
diff --git a/app-packages/hbase-win/appConfig.json
b/app-packages/hbase-win/appConfig.json
index 6f07d34..5b97e69 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -5,6 +5,7 @@
"global": {
"application.def":
".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
+"java_home": "C:\\java",
"site.global.app_user": "hadoop",
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7c78605/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bb98588..6976f50 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -567,8 +567,8 @@ public class AgentProviderService extends
AbstractProviderService implements
response.setRestartEnabled(true);
}
} catch (SliderException e) {
- componentStatus.applyCommandResult(CommandResult.FAILED, command);
log.warn("Component instance failed operation.", e);
+ componentStatus.applyCommandResult(CommandResult.FAILED, command);
}
log.debug("Heartbeat response: " + response);
[16/19] git commit: SLIDER-149 java 5 template pain
SLIDER-149 java 5 template pain
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b26863ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b26863ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b26863ab
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b26863ab63b20c6f008728dc0c5827a05b37c5cb
Parents: a06d5ed
Author: Steve Loughran
Authored: Tue Sep 23 21:53:57 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 23 21:53:57 2014 +0100
--
.../main/java/org/apache/slider/core/persist/JsonSerDeser.java | 2 +-
.../slider/server/services/curator/RegistryBinderService.java | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b26863ab/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 97be41d..f77d6df 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -56,7 +56,7 @@ public class JsonSerDeser {
/**
* Create an instance bound to a specific type
- * @param classType
+ * @param classType class type
*/
public JsonSerDeser(Class classType) {
this.classType = classType;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b26863ab/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
index b3e2ff2..dbdcee4 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -27,7 +27,6 @@ import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
import org.apache.curator.x.discovery.ServiceType;
import org.apache.curator.x.discovery.UriSpec;
-import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -56,8 +55,8 @@ public class RegistryBinderService extends
CuratorService {
private final Map> entries =
new HashMap>();
- private JsonSerDeser> deser =
-new
JsonSerDeser>(CuratorServiceInstance.class);
+ private JsonSerDeser deser =
+new JsonSerDeser(CuratorServiceInstance.class);
/**
* Create an instance
[05/19] git commit: SLIDER-444. Minor corrections to default app packages
SLIDER-444. Minor corrections to default app packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ffd2d57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ffd2d57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ffd2d57
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0ffd2d57ddb1fd18d1f5f3b0b069cea1339e4905
Parents: a025879
Author: Sumit Mohanty
Authored: Thu Sep 18 20:32:11 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 18 20:32:11 2014 -0700
--
app-packages/hbase/ganglia_metrics.json | 38 +
app-packages/storm/appConfig.json | 2 +
app-packages/storm/metainfo.xml | 22 +-
app-packages/storm/package/scripts/params.py| 4 ++
app-packages/storm/package/scripts/service.py | 40 --
.../storm/package/scripts/status_params.py | 2 +
.../storm/package/templates/config.yaml.j2 | 9
app-packages/storm/resources.json | 4 ++
slider-assembly/src/conf/slider-client.xml | 44 ++--
9 files changed, 129 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ffd2d57/app-packages/hbase/ganglia_metrics.json
--
diff --git a/app-packages/hbase/ganglia_metrics.json
b/app-packages/hbase/ganglia_metrics.json
new file mode 100644
index 000..da73d48
--- /dev/null
+++ b/app-packages/hbase/ganglia_metrics.json
@@ -0,0 +1,38 @@
+{
+"Component": {
+"HBASE_REGIONSERVER": {
+"readRequestsCount": {
+"metric": "regionserver.Server.readRequestCount",
+"pointInTime": false,
+"temporal": true
+},
+"regions": {
+"metric": "regionserver.Server.regionCount",
+"pointInTime": false,
+"temporal": true
+},
+"flushQueueSize": {
+"metric": "regionserver.Server.flushQueueLength",
+"pointInTime": false,
+"temporal": true
+}
+},
+"HBASE_MASTER": {
+"cluster_requests": {
+"metric": "master.Server.clusterRequests",
+"pointInTime": false,
+"temporal": true
+},
+"splitTime_avg_time": {
+"metric": "master.FileSystem.HlogSplitTime_mean",
+"pointInTime": false,
+"temporal": true
+},
+"splitSize_avg_time": {
+"metric": "master.FileSystem.HlogSplitSize_mean",
+"pointInTime": false,
+"temporal": true
+}
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ffd2d57/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index bb607e9..4af092d 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -16,6 +16,8 @@
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
"site.global.ganglia_server_port": "8668",
+"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
+"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ffd2d57/app-packages/storm/metainfo.xml
--
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 1914551..2bf6ddc 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -29,6 +29,10 @@
QuickLinks
+app.jmx
+
http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary
+
+
app.monitor
http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}
@@ -51,7 +55,7 @@
NIMBUS-START
-
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
+
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
@@ -66,6 +70,10 @@
STORM_UI_SERVER-START
NIMBUS-STARTED
+
+STORM_REST_API-START
+
NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED
+
@@ -122,6 +130,1
[07/19] git commit: SLIDER-434. Modify default application package locations to the default upload location
SLIDER-434. Modify default application package locations to the default upload
location
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0d41051a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0d41051a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0d41051a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0d41051a40709071336e1f2331eb0167edf5cd1a
Parents: 5ab149c
Author: Sumit Mohanty
Authored: Sat Sep 20 23:40:06 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Sep 20 23:40:06 2014 -0700
--
app-packages/accumulo/appConfig.json| 2 +-
app-packages/accumulo/pom.xml | 1 +
.../command-logger/slider-pkg/appConfig.json| 2 +-
app-packages/hbase-win/appConfig.json | 2 +-
app-packages/hbase/appConfig.json | 2 +-
app-packages/memcached-win/appConfig.json | 2 +-
app-packages/memcached/appConfig.json | 2 +-
app-packages/storm/appConfig.json | 2 +-
.../test_command_log/appConfig.json | 4 +--
.../test_command_log/appConfig_fast_no_reg.json | 3 +-
.../test_command_log/appConfig_no_hb.json | 4 +--
slider-funtest/pom.xml | 1 +
.../framework/AgentCommandTestBase.groovy | 31 +++-
.../funtest/lifecycle/AMFailuresIT.groovy | 1 -
.../lifecycle/AgentClusterLifecycleIT.groovy| 1 -
.../funtest/lifecycle/AgentFailures2IT.groovy | 1 -
.../funtest/lifecycle/AgentFailuresIT.groovy| 1 -
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 2 +-
.../lifecycle/ClusterBuildDestroyIT.groovy | 1 -
19 files changed, 24 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/accumulo/appConfig.json
--
diff --git a/app-packages/accumulo/appConfig.json
b/app-packages/accumulo/appConfig.json
index d6f849c..28fb637 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "${app.package.name}.zip",
+"application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
"java_home": "/usr/lib/jvm/java",
"site.global.app_user": "${app.user}",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index c7d007e..02a0a72 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -190,6 +190,7 @@
${slider.bin.dir}
${test.app.pkg.dir}
${app.package.name}.zip
+ACCUMULO
${test.app.resources.dir}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/command-logger/slider-pkg/appConfig.json
--
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json
b/app-packages/command-logger/slider-pkg/appConfig.json
index 791043f..73bc7fa 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "apache-slider-command-logger.zip",
+"application.def":
".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/hbase-win/appConfig.json
--
diff --git a/app-packages/hbase-win/appConfig.json
b/app-packages/hbase-win/appConfig.json
index 8261f8c..94f1343 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
-"application.def": "${app.package.name}.zip",
+"application.def": ".slider/package/HBASE/${app.package.name}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/hbase/appConfig.json
--
diff --git a/app-packages/hbase/appConfig.json
b/app-packages/hbase/appConfig.json
index 374
[02/19] git commit: SLIDER-278 add optional custom app repo via flag -Dprivate.repo.url=...
SLIDER-278 add optional custom app repo via flag -Dprivate.repo.url=...
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7661f481
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7661f481
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7661f481
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7661f48143da52d306518839a3785c9a1a14347f
Parents: 2d6f483
Author: Billie Rinaldi
Authored: Thu Sep 18 07:52:23 2014 -0700
Committer: Billie Rinaldi
Committed: Thu Sep 18 07:52:23 2014 -0700
--
pom.xml | 23 +++
1 file changed, 23 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7661f481/pom.xml
--
diff --git a/pom.xml b/pom.xml
index df8599f..9dd400e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1436,6 +1436,29 @@
+
+
+ private-repo
+
+
+
+ private.repo.url
+
+
+
+
+
+true
+
+
+false
+
+ private-repo
+ Private Repo
+ ${private.repo.url}
+
+
+
[2/5] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8c7f6b23 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8c7f6b23 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8c7f6b23 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 8c7f6b23d5cf0202718b38d182ff5624f4bd8955 Parents: 8b2d4d5 94df22c Author: Steve Loughran Authored: Wed Sep 24 15:00:51 2014 +0100 Committer: Steve Loughran Committed: Wed Sep 24 15:00:51 2014 +0100 -- --
[1/5] git commit: SLIDER-453 AM queue processing goes live before init complete
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 8b2d4d5a1 ->
da814b744
SLIDER-453 AM queue processing goes live before init complete
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/94df22c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/94df22c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/94df22c9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 94df22c92e48859fbc4b3a560981a6a70cc64b54
Parents: 937e0a7
Author: Steve Loughran
Authored: Tue Sep 23 21:59:01 2014 +0100
Committer: Steve Loughran
Committed: Tue Sep 23 21:59:42 2014 +0100
--
.../server/appmaster/SliderAppMaster.java | 29 ++--
.../slider/server/appmaster/state/AppState.java | 5 +++-
2 files changed, 30 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 08d0049..365eeab 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -283,6 +283,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
*/
private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
+ /**
+ * Flag set during the init process
+ */
+ private final AtomicBoolean initCompleted = new AtomicBoolean(false);
+
private volatile boolean success = true;
/**
@@ -419,11 +424,18 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+ }
+
+ /**
+ * Start the queue processing
+ */
+ private void startQueueProcessing() {
+log.info("Queue Processing started");
executorService.execute(actionQueues);
executorService.execute(new QueueExecutor(this, actionQueues));
}
- /* === */
+/* === */
/* RunService methods called from ServiceLauncher */
/* === */
@@ -801,7 +813,15 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
+// declare the cluster initialized
+log.info("Application Master Initialization Completed");
+initCompleted.set(true);
+
+// start handling any scheduled events
+
+startQueueProcessing();
// Start the Slider AM provider
+
sliderAMProvider.start();
// launch the real provider; this is expected to trigger a callback that
@@ -1210,8 +1230,11 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
log.error("Role instance {} failed ", ri);
}
- getProviderService().notifyContainerCompleted(containerId);
- queue(new UnregisterComponentInstance(containerId, 0,
TimeUnit.MILLISECONDS));
+ // known nodes trigger notifications
+ if(!result.unknownNode) {
+getProviderService().notifyContainerCompleted(containerId);
+queue(new UnregisterComponentInstance(containerId, 0,
TimeUnit.MILLISECONDS));
+ }
}
reviewRequestAndReleaseNodes("onContainersCompleted");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 57952e7..abb6fe8 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1256,13 +1256,14 @@ public class AppState {
}
/**
- * This is a very small class to send a triple result back from
+ * This is a very small class to send a multiple result back from
* the completion operation
*/
public static class NodeCompletionResult {
public boolean surplusNode = false;
public RoleInstance roleInstance;
public boolean containerFailed;
+public boolean unknownN
[5/5] git commit: SLIDER-149 REST registry interface; in sync with latest changes of the code
SLIDER-149 REST registry interface; in sync with latest changes of the code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/da814b74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/da814b74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/da814b74
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: da814b7440ea8aeb21b9811df53b93b0eddece33
Parents: 44e6165
Author: Steve Loughran
Authored: Thu Sep 25 20:43:50 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 25 20:43:50 2014 +0100
--
.../org/apache/slider/client/SliderClient.java | 11 +-
.../core/exceptions/ExceptionConverter.java | 16 +-
.../docstore/PublishedConfiguration.java| 4 +-
.../core/registry/info/RegisteredDocument.java | 33 ---
.../core/registry/info/RegisteredEndpoint.java | 1 +
.../core/registry/info/RegistryFields.java | 34 ---
.../core/registry/info/RegistryNaming.java | 51 -
.../slider/core/registry/info/RegistryView.java | 1 +
.../server/appmaster/SliderAppMaster.java | 21 +-
.../web/rest/registry/PathEntryResource.java| 51 +
.../web/rest/registry/RegistryResource.java | 81 ---
.../server/services/curator/CuratorHelper.java | 1 +
.../TestStandaloneYarnRegistryAM.groovy | 24 ++-
.../registry/curator/TestLocalRegistry.groovy | 164 --
.../curator/TestRegistryRestResources.groovy| 195 -
.../curator/TestServiceInstanceSerDeser.groovy | 122 ---
.../rest/registry/PathEntryMarshalling.groovy | 28 +++
.../registry/TestRegistryRestMarshalling.groovy | 47
.../registry/TestRegistryRestResources.groovy | 213 +++
slider-core/src/test/resources/log4j.properties | 3 +-
.../appmaster/web/rest/registry/sample.json | 76 +++
.../funtest/FunctionalHBaseClusterIT.groovy | 7 +-
.../minicluster/live/TestHBaseMaster.groovy | 2 +-
23 files changed, 526 insertions(+), 660 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e68a7cb..de27dcd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -44,6 +44,7 @@ import
org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import static
org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*;
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
@@ -2453,9 +2454,9 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.id);
+ log.info("{}", instance.yarn_id);
} else {
- log.info("{}: ", instance.id);
+ log.info("{}: ", instance.yarn_id);
logEndpoints(instance);
}
}
@@ -2599,7 +2600,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @param serviceType service type
* @param id instance ID
* @return instance data
- * @throws UnknownApplicationInstanceException no match
+ * @throws UnknownApplicationInstanceException no path or service record
+ * at the end of the path
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
@@ -2609,8 +2611,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return getRegistryOperations().resolve(
servicePath(currentUser(),
serviceType, id));
+ // TODO JDK7 SWITCH
} catch (PathNotFoundException e) {
throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
+} catch (NoRecordException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
--
diff --git
a/slider-core/src/m
[3/5] SLIDER-149 migration off curator-x-discovery: read only REST API (work in progress)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 68cac7b..e04f9cb 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -96,11 +96,6 @@ class TestStandaloneAgentAM extends
AgentMiniClusterTestBase {
assert instance != null
//switch to the slider ZK-based registry
-
-describe "service registry names"
-SliderRegistryService registry = client.registry
-def names = registry.getServiceTypes();
-dumpRegistryServiceTypes(names)
describe "service registry instance IDs"
def instanceIds = client.listRegisteredSliderInstances()
@@ -108,14 +103,6 @@ class TestStandaloneAgentAM extends
AgentMiniClusterTestBase {
log.info("number of instanceIds: ${instanceIds.size()}")
instanceIds.each { String it -> log.info(it) }
-describe "service registry slider instances"
-List> instances =
-client.listRegistryInstances()
-instances.each { CuratorServiceInstance svc ->
- log.info svc.toString()
-}
-describe "end list service registry slider instances"
-
describe "Yarn registry"
def yarnRegistry = client.registryOperations
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index e88fbe8..65bd077 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -20,17 +20,18 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsClient
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import static
org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
@@ -73,7 +74,7 @@ class TestStandaloneYarnRegistryAM extends
AgentMiniClusterTestBase {
registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
// verify the cluster has the YARN reg service live
-def rmRegistryService =
miniCluster.getResourceManager(0).getRMContext().registry
+def rmRegistryService =
miniCluster.getResourceManager(0).RMContext.registry
assert rmRegistryService
ServiceLauncher launcher
@@ -124,48 +125,47 @@ class TestStandaloneYarnRegistryAM extends
AgentMiniClusterTestBase {
// sleep to allow registration to complete
sleep(5000)
-
-
+describe "service registry names"
+def registryService = client.registryOperations
+
+RegistryOperationsClient registryOperationsClient =
+registryService as RegistryOperationsClient
try {
- def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ def yarnRegistryDump = registryOperationsClient.dumpPath()
log.info("yarn service registry: \n${yarnRegistryDump}\n")
} catch (IOException ignored) {
}
-
-describe "service registry names"
-def registryService = client.registryOperations
def self = currentUser()
-RegistryPathStatus[] serviceTypes = registryService.list(userPath(self))
-dumpArray(serviceTypes)
+List serviceTypes =
registryService.listFull(homePathForUser(self))
+dumpCollection(servi
[4/5] git commit: SLIDER-149 migration off curator-x-discovery: read only REST API (work in progress)
SLIDER-149 migration off curator-x-discovery: read only REST API (work in
progress)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/44e61659
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/44e61659
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/44e61659
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 44e61659f0c9cefb16b2cbc1f63447ee4c96c4fc
Parents: 8c7f6b2
Author: Steve Loughran
Authored: Thu Sep 25 10:50:08 2014 +0100
Committer: Steve Loughran
Committed: Thu Sep 25 10:50:08 2014 +0100
--
.../funtest/accumulo/AccumuloBasicIT.groovy | 8 +-
.../org/apache/slider/client/SliderClient.java | 271 ---
.../slider/client/SliderClusterOperations.java | 1 -
.../slider/core/launch/AbstractLauncher.java| 8 +-
.../core/launch/JavaCommandLineBuilder.java | 4 +-
.../registry/info/CustomRegistryConstants.java | 1 +
.../core/registry/info/ServiceInstanceData.java | 1 +
.../registry/retrieve/RegistryRetriever.java| 10 -
.../providers/AbstractProviderService.java | 46 ++--
.../slider/providers/ProviderService.java | 11 +-
.../providers/agent/AgentProviderService.java | 22 +-
.../slideram/SliderAMProviderService.java | 37 +--
.../server/appmaster/SliderAppMaster.java | 82 ++
.../server/appmaster/web/SliderAMWebApp.java| 22 +-
.../slider/server/appmaster/web/WebAppApi.java | 4 +
.../server/appmaster/web/WebAppApiImpl.java | 17 +-
.../appmaster/web/rest/AMWebServices.java | 9 +
.../server/appmaster/web/rest/RestPaths.java| 4 +-
.../web/rest/registry/RegistryResource.java | 130 +
.../server/services/curator/CuratorHelper.java | 11 -
.../server/services/curator/CuratorService.java | 1 +
.../curator/CuratorServiceInstance.java | 1 +
.../curator/CuratorServiceInstances.java| 1 +
.../server/services/curator/CuratorUriSpec.java | 1 +
.../services/curator/RegistryBinderService.java | 1 +
.../registry/RegistryDiscoveryContext.java | 1 +
.../registry/RegistryRestResources.java | 1 +
.../registry/RegistryServiceConstants.java | 1 +
.../registry/RegistryViewForProviders.java | 48
.../registry/SliderRegistryService.java | 8 +-
.../utility/AbstractSliderLaunchedService.java | 52 +---
.../YarnRegistryViewForProviders.java | 32 ++-
.../standalone/TestStandaloneAgentAM.groovy | 13 -
.../TestStandaloneYarnRegistryAM.groovy | 46 ++--
.../model/mock/MockProviderService.groovy | 4 -
.../view/TestClusterSpecificationBlock.groovy | 7 +-
.../web/view/TestContainerStatsBlock.groovy | 7 +-
.../appmaster/web/view/TestIndexBlock.groovy| 8 +-
.../apache/slider/test/SliderTestUtils.groovy | 13 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 2 +-
.../publisher/TestAgentProviderService.java | 5 +-
.../minicluster/live/TestHBaseMaster.groovy | 33 +--
.../minicluster/live/TestTwoLiveClusters.groovy | 35 ++-
44 files changed, 409 insertions(+), 614 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 475c4e3..613769e 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -25,6 +25,7 @@ import org.apache.hadoop.security.ProviderUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.alias.CredentialProvider
import org.apache.hadoop.security.alias.CredentialProviderFactory
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
@@ -170,10 +171,9 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase
{
int tries = 5
while (true) {
try {
-CuratorServiceInstance instance =
- sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE,
clusterName)
-ServiceInstanceData serviceInstanceData = instance.payload
-RegistryRetriever retriever = new
RegistryRetriever(serviceInstanceDa
git commit: SLIDER-149 purged all references/uses of the curator service discovery and the original slider registry code.
Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry da814b744 -> 2154c7530 SLIDER-149 purged all references/uses of the curator service discovery and the original slider registry code. Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2154c753 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2154c753 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2154c753 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 2154c753023fb9f3372945277f5a63905a5b0c6d Parents: da814b7 Author: Steve Loughran Authored: Thu Sep 25 21:15:36 2014 +0100 Committer: Steve Loughran Committed: Thu Sep 25 21:15:36 2014 +0100 -- .../funtest/accumulo/AccumuloBasicIT.groovy | 3 +- slider-core/pom.xml | 10 - .../org/apache/slider/client/SliderClient.java | 22 -- .../slider/core/registry/info/RegistryView.java | 42 --- .../core/registry/info/ServiceInstanceData.java | 140 - .../slideram/SliderAMClientProvider.java| 6 - .../server/appmaster/web/SliderAMWebApp.java| 8 - .../server/services/curator/CuratorHelper.java | 133 .../server/services/curator/CuratorService.java | 115 --- .../curator/CuratorServiceInstance.java | 78 - .../curator/CuratorServiceInstances.java| 42 --- .../server/services/curator/CuratorUriSpec.java | 58 .../services/curator/RegistryBinderService.java | 315 --- .../registry/RegistryDiscoveryContext.java | 37 --- .../registry/RegistryRestResources.java | 179 --- .../registry/RegistryServiceConstants.java | 27 -- .../registry/SliderRegistryService.java | 95 -- .../YarnRegistryViewForProviders.java | 30 +- .../standalone/TestStandaloneAgentAM.groovy | 4 - .../apache/slider/test/SliderTestUtils.groovy | 5 - .../slider/providers/hbase/HBaseKeys.java | 2 +- .../providers/hbase/HBaseProviderService.java | 33 +- 22 files changed, 47 insertions(+), 1337 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy -- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index 613769e..5d4468c 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy @@ -34,10 +34,9 @@ import org.apache.slider.common.tools.SliderUtils import org.apache.slider.core.conf.ConfTree import org.apache.slider.core.persist.ConfTreeSerDeser import org.apache.slider.core.registry.docstore.PublishedConfiguration -import org.apache.slider.core.registry.info.ServiceInstanceData import org.apache.slider.core.registry.retrieve.RegistryRetriever import org.apache.slider.funtest.framework.SliderShell -import org.apache.slider.server.services.curator.CuratorServiceInstance + import org.junit.Before import org.junit.Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/pom.xml -- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 8263199..d570606 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -346,16 +346,6 @@ - org.apache.curator - curator-x-discovery - - - - org.apache.curator - curator-x-discovery-server - - - org.apache.zookeeper zookeeper http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index de27dcd..63bf9c2 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -109,8 +109,6 @@ import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.
git commit: SLIDER-149; in sync with registry code. HBase provider test working
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 2154c7530 ->
46fb5dbdb
SLIDER-149; in sync with registry code. HBase provider test working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46fb5dbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46fb5dbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46fb5dbd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 46fb5dbdb35c5b072771a394ec339f1bd7e8f56c
Parents: 2154c75
Author: Steve Loughran
Authored: Fri Sep 26 14:18:06 2014 +0100
Committer: Steve Loughran
Committed: Fri Sep 26 14:18:06 2014 +0100
--
.../registry/retrieve/RegistryRetriever.java| 43 +---
.../server/appmaster/SliderAppMaster.java | 3 +-
.../YarnRegistryViewForProviders.java | 17 +---
.../TestStandaloneYarnRegistryAM.groovy | 2 +-
.../providers/hbase/HBaseProviderService.java | 12 --
5 files changed, 51 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 1d11ef3..df1d675 100644
---
a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++
b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -25,8 +25,10 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import
org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.exceptions.RegistryIOException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.common.tools.SliderUtils;
@@ -68,25 +70,37 @@ public class RegistryRetriever {
this.internalConfigurationURL = internalConfigurationURL;
}
- public RegistryRetriever(ServiceRecord record) throws InvalidRecordException
{
+ /**
+ * Retrieve from a service by locating the
+ * exported {@link CustomRegistryConstants#PUBLISHER_CONFIGURATIONS_API}
+ * and working off it.
+ * @param record service record
+ * @throws RegistryIOException the address type of the endpoint does
+ * not match that expected (i.e. not a list of URLs), missing endpoint...
+ */
+ public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
Endpoint internal = record.getInternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-List addresses = RegistryTypeUtils.retrieveAddressesUriType(
-internal);
-if (addresses != null && !addresses.isEmpty()) {
- internalConfigurationURL = addresses.get(0);
-} else {
- internalConfigurationURL = "";
+String url = null;
+if (internal != null) {
+ List addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+url = addresses.get(0);
+ }
}
+internalConfigurationURL = url;
Endpoint external = record.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-
-addresses = RegistryTypeUtils.retrieveAddressesUriType(external);
-if (addresses != null && !addresses.isEmpty()) {
- externalConfigurationURL = addresses.get(0);
-} else {
- externalConfigurationURL = "";
+url = null;
+if (external != null) {
+ List addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+url = addresses.get(0);
+ }
}
+externalConfigurationURL = url;
}
/**
@@ -166,7 +180,8 @@ public class RegistryRetriever {
@Override
public String toString() {
return super.toString()
- + " - external " + externalConfigurationURL;
+ + ": internal URL: \"" + internalConfigurationURL
+ + "\"; external \"" + externalConfigurationURL +"\"";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/s
git commit: SLIDER-149 purge RegisteredEndpoint code, and so all traces of the original registry
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 46fb5dbdb ->
65e7684de
SLIDER-149 purge RegisteredEndpoint code, and so all traces of the original
registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/65e7684d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/65e7684d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/65e7684d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 65e7684de15c6340caff0381ceb99f2a12ca3f72
Parents: 46fb5db
Author: Steve Loughran
Authored: Fri Sep 26 14:44:38 2014 +0100
Committer: Steve Loughran
Committed: Fri Sep 26 14:44:38 2014 +0100
--
.../core/registry/info/RegisteredEndpoint.java | 180 ---
.../providers/agent/AgentProviderService.java | 2 +-
.../server/appmaster/state/RoleInstance.java| 29 +--
3 files changed, 16 insertions(+), 195 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/65e7684d/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
deleted file mode 100644
index 2351e06..000
---
a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
-import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Deprecated
-public class RegisteredEndpoint {
-
- // standard types
-
- /**
- * URL: {@value}
- */
- public static final String TYPE_URL = AddressTypes.ADDRESS_URI;
-
-
- /**
- * "hostname:port" pair: {@value}
- */
- public static final String TYPE_INETADDRESS =
AddressTypes.ADDRESS_HOSTNAME_AND_PORT;
-
- /**
- * simple path string: {@value}
- */
- public static final String TYPE_PATH = AddressTypes.ADDRESS_PATH;
-
- // standard protocols
- /**
- * Classic "Writable" Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_RPC =
ProtocolTypes.PROTOCOL_HADOOP_IPC;
-
- /**
- * Protocol buffer based Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_PROTOBUF =
ProtocolTypes.PROTOCOL_HADOOP_IPC_PROTOBUF;
-
- /**
- * The address -format is driven by the type entry
- */
- public String address;
-
- /**
- * Protocol
- */
- public String protocol = "";
-
- public String type = "";
-
- /**
- * Human readable type
- */
- public String description = "";
-
- public RegisteredEndpoint() {
- }
-
- public RegisteredEndpoint(String address,
-String protocol,
-String type,
-String description) {
-this.address = address;
-this.protocol = protocol;
-this.type = type;
-this.description = description;
- }
-
- /**
- * Build an endpoint instance from a URI, extracting
- * the protocol from it
- * @param uri URI to set the value to
- * @param description description
- */
- public RegisteredEndpoint(URI uri,
-String description) {
-
-this.address = uri.toString();
-this.protocol = uri.getScheme();
-this.type
[6/9] git commit: SLIDER-456 switch accumulo version to 1.6.1
SLIDER-456 switch accumulo version to 1.6.1 Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ed960d33 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ed960d33 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ed960d33 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: ed960d33c4baa948e48a18c324556912e9d5517e Parents: 5470c47 Author: Billie Rinaldi Authored: Thu Sep 25 06:58:46 2014 -0700 Committer: Billie Rinaldi Committed: Thu Sep 25 06:58:46 2014 -0700 -- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed960d33/pom.xml -- diff --git a/pom.xml b/pom.xml index 9dd400e..b31d476 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ 2.6.0-SNAPSHOT 0.98.4-hadoop2 -1.6.1-SNAPSHOT +1.6.1
[4/9] git commit: SLIDER-116. Log handling for long-lived applications (Gour Saha via smohanty)
SLIDER-116. Log handling for long-lived applications (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34837ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34837ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34837ce
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a34837ce1d0431c960f956251eb57e23054253d1
Parents: 314c3cc
Author: Sumit Mohanty
Authored: Wed Sep 24 13:50:52 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Sep 24 13:51:00 2014 -0700
--
app-packages/accumulo/resources.json| 3 +
.../accumulo/src/test/resources/resources.json | 3 +
app-packages/hbase-win/resources.json | 3 +
.../hbase/src/test/resources/resources.json | 3 +
app-packages/storm/resources.json | 3 +
.../org/apache/slider/api/ResourceKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 2 +
.../slider/core/launch/AbstractLauncher.java| 51 +
.../slider/core/launch/AppMasterLauncher.java | 4 +-
.../core/launch/TestAppMasterLauncher.java | 112 +++
10 files changed, 186 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/resources.json
--
diff --git a/app-packages/accumulo/resources.json
b/app-packages/accumulo/resources.json
index f876901..f9cae60 100644
--- a/app-packages/accumulo/resources.json
+++ b/app-packages/accumulo/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/src/test/resources/resources.json
--
diff --git a/app-packages/accumulo/src/test/resources/resources.json
b/app-packages/accumulo/src/test/resources/resources.json
index 2711ba3..cfd9b3f 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase-win/resources.json
--
diff --git a/app-packages/hbase-win/resources.json
b/app-packages/hbase-win/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase-win/resources.json
+++ b/app-packages/hbase-win/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase/src/test/resources/resources.json
--
diff --git a/app-packages/hbase/src/test/resources/resources.json
b/app-packages/hbase/src/test/resources/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase/src/test/resources/resources.json
+++ b/app-packages/hbase/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/storm/resources.json
--
diff --git a/app-packages/storm/resources.json
b/app-packages/storm/resources.json
index 2805a3b..acecd91 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -3,6 +3,9 @@
"metadata" : {
},
"global" : {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index b542f1a..56961c9 100644
--- a/slider-core/src/main/
[1/9] git commit: SLIDER-451. slider install-package could use better error messages
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 65e7684de ->
a3e5b372c
SLIDER-451. slider install-package could use better error messages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/47b4a9c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/47b4a9c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/47b4a9c6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 47b4a9c606d6e7064c690f60ab9f456d8ec6ac75
Parents: 94df22c
Author: Sumit Mohanty
Authored: Tue Sep 23 21:07:55 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Sep 23 21:07:55 2014 -0700
--
.../src/main/java/org/apache/slider/client/SliderClient.java | 6 +++---
.../slider/agent/actions/TestActionInstallPackage.groovy | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47b4a9c6/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 221fc43..129f851 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -623,11 +623,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
Path srcFile = null;
if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) {
- throw new BadCommandArgumentsException("A valid application name is
required.");
+ throw new BadCommandArgumentsException("A valid application type name is
required (e.g. HBASE).");
}
if (null == installPkgInfo.packageURI ||
installPkgInfo.packageURI.length() == 0) {
- throw new BadCommandArgumentsException("A valid application package is
required.");
+ throw new BadCommandArgumentsException("A valid application package
location required.");
} else {
File pkgFile = new File(installPkgInfo.packageURI);
if (!pkgFile.exists() || pkgFile.isDirectory()) {
@@ -646,7 +646,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
if (sliderFileSystem.getFileSystem().exists(fileInFs) &&
!installPkgInfo.replacePkg) {
throw new BadCommandArgumentsException("Pkg exists at " +
fileInFs.toUri().toString() +
- ". Use --replacePkg to
overwrite.");
+ ". Use --replacepkg to
overwrite.");
}
sliderFileSystem.getFileSystem().copyFromLocalFile(false,
installPkgInfo.replacePkg, srcFile, fileInFs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47b4a9c6/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
index 88f1298..064c7a9 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
@@ -56,7 +56,7 @@ class TestActionInstallPackage extends
AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " +
launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("A valid application name is required")
+ assert e.message.contains("A valid application type name is required
(e.g. HBASE)")
}
}
@@ -74,7 +74,7 @@ class TestActionInstallPackage extends
AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " +
launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("A valid application package is required.")
+ assert e.message.contains("A valid application package location
required")
}
}
@@ -103,7 +103,7 @@ class TestActionInstallPackage extends
AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " +
launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("Use --replacePkg to overwrite")
+ assert e.message.contains("Use --replacepkg to overwrite")
}
}
[7/9] git commit: SLIDER-450. The template and resources json files embedded within the app package should be named default and include comments to that effect
SLIDER-450. The template and resources json files embedded within the app
package should be named default and include comments to that effect
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/772759c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/772759c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/772759c3
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 772759c305bd11e940a4fa0f4f2d7994d1581471
Parents: ed960d3
Author: Sumit Mohanty
Authored: Thu Sep 25 22:25:49 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 25 22:25:49 2014 -0700
--
app-packages/accumulo/README.txt| 9 ++--
app-packages/app-pkg-template/README.txt| 7 ++-
.../app-pkg-template/appConfig-default.json | 21 +
app-packages/app-pkg-template/appConfig.json| 21 -
.../app-pkg-template/resources-default.json | 16 +++
app-packages/app-pkg-template/resources.json| 16 ---
app-packages/hbase-win/README.txt | 8 ++--
app-packages/hbase/README.txt | 7 ++-
app-packages/memcached-win/README.txt | 7 ++-
.../memcached-win/appConfig-default.json| 24 ++
app-packages/memcached-win/appConfig.json | 24 --
.../memcached-win/resources-default.json| 16 +++
app-packages/memcached-win/resources.json | 16 ---
app-packages/memcached/README.txt | 7 ++-
app-packages/memcached/appConfig-default.json | 24 ++
app-packages/memcached/appConfig.json | 24 --
app-packages/memcached/resources-default.json | 16 +++
app-packages/memcached/resources.json | 16 ---
app-packages/storm/README.txt | 7 ++-
app-packages/storm/appConfig-default.json | 46
app-packages/storm/appConfig.json | 46
app-packages/storm/resources-default.json | 34 +++
app-packages/storm/resources.json | 34 ---
app-packages/storm/src/assembly/storm.xml | 4 +-
24 files changed, 222 insertions(+), 228 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/accumulo/README.txt
--
diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt
index f75814c..3a857ee 100644
--- a/app-packages/accumulo/README.txt
+++ b/app-packages/accumulo/README.txt
@@ -32,10 +32,9 @@ slider version, and in the second case it will match the
pkg.version
Verify the content using
zip -Tv slider-accumulo-app-package*.zip
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be
packaged.
+These files are included as reference configuration for Slider apps and are
suitable
+for a one-node cluster.
In the maven packaging case, the version of Accumulo used for the app package
can be adjusted by adding a flag such as
@@ -45,7 +44,7 @@ can be adjusted by adding a flag such as
package are designed for Accumulo 1.6.0 only and may need to be modified to be
applicable for other versions of the app package.
-Note also that the sample appConfig.json provided only works with Accumulo 1.6,
+Note also that the sample appConfig-default.json provided only works with
Accumulo 1.6,
while for Accumulo 1.5 the instance.volumes property must be replaced with
instance.dfs.dir (and it cannot use the provided variable ${DEFAULT_DATA_DIR}
which is an HDFS URI).
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/README.txt
--
diff --git a/app-packages/app-pkg-template/README.txt
b/app-packages/app-pkg-template/README.txt
index 00dfdbc..266f34f 100644
--- a/app-packages/app-pkg-template/README.txt
+++ b/app-packages/app-pkg-template/README.txt
@@ -28,7 +28,6 @@ Create a zip package at the root of the package (/app-package
Verify the content using
zip -Tv myapp-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they
work
-well as the default configuration for Slider apps. So its advisable that when
you
-create an application package for Slider, include sample/default
resources.json and
-appConfig.json for a one-node Yarn cluster.
+appConfig-default.json and resources-defa
[9/9] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3e5b372 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3e5b372 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3e5b372 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: a3e5b372cc257b1df65ad35565eb743e0775a7af Parents: 65e7684 43f9a94 Author: Steve Loughran Authored: Fri Sep 26 14:48:14 2014 +0100 Committer: Steve Loughran Committed: Fri Sep 26 14:50:02 2014 +0100 -- app-packages/accumulo/README.txt| 9 +- app-packages/accumulo/resources.json| 3 + .../accumulo/src/test/resources/resources.json | 3 + app-packages/app-pkg-template/README.txt| 7 +- .../app-pkg-template/appConfig-default.json | 21 app-packages/app-pkg-template/appConfig.json| 21 .../app-pkg-template/resources-default.json | 16 +++ app-packages/app-pkg-template/resources.json| 16 --- app-packages/hbase-win/README.txt | 8 +- app-packages/hbase-win/appConfig-default.json | 38 +++ app-packages/hbase-win/appConfig.json | 38 --- app-packages/hbase-win/resources-default.json | 24 app-packages/hbase-win/resources.json | 21 app-packages/hbase-win/src/assembly/hbase.xml | 4 +- app-packages/hbase/README.txt | 7 +- app-packages/hbase/appConfig-default.json | 47 app-packages/hbase/appConfig.json | 47 app-packages/hbase/pom.xml | 4 +- app-packages/hbase/resources-default.json | 36 ++ app-packages/hbase/resources.json | 36 -- app-packages/hbase/src/assembly/hbase.xml | 4 +- .../src/test/resources/resources-default.json | 24 .../hbase/src/test/resources/resources.json | 21 app-packages/memcached-win/README.txt | 7 +- .../memcached-win/appConfig-default.json| 24 app-packages/memcached-win/appConfig.json | 24 .../memcached-win/resources-default.json| 16 +++ app-packages/memcached-win/resources.json | 16 --- app-packages/memcached/README.txt | 7 +- app-packages/memcached/appConfig-default.json | 24 app-packages/memcached/appConfig.json | 24 app-packages/memcached/resources-default.json | 16 +++ app-packages/memcached/resources.json | 16 --- app-packages/storm/README.txt | 7 +- app-packages/storm/appConfig-default.json | 46 app-packages/storm/appConfig.json | 46 app-packages/storm/resources-default.json | 34 ++ app-packages/storm/resources.json | 31 - app-packages/storm/src/assembly/storm.xml | 4 +- pom.xml | 2 +- .../python/agent/CustomServiceOrchestrator.py | 15 ++- .../src/main/python/agent/ProcessHelper.py | 5 +- slider-agent/src/main/python/agent/main.py | 13 ++- .../agent/TestCustomServiceOrchestrator.py | 20 +++- .../org/apache/slider/api/ResourceKeys.java | 3 + .../org/apache/slider/client/SliderClient.java | 8 +- .../slider/core/launch/AbstractLauncher.java| 52 + .../slider/core/launch/AppMasterLauncher.java | 4 +- .../slider/providers/agent/AgentKeys.java | 2 +- .../providers/agent/AgentProviderService.java | 6 +- .../web/rest/agent/ExecutionCommand.java| 1 + .../actions/TestActionInstallPackage.groovy | 6 +- .../core/launch/TestAppMasterLauncher.java | 112 +++ .../agent/TestAgentProviderService.java | 2 +- 54 files changed, 624 insertions(+), 424 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e5b372/pom.xml -- diff --cc pom.xml index fed95a8,b31d476..bfc52bd --- a/pom.xml +++ b/pom.xml @@@ -136,10 -136,10 +136,10 @@@ -2.6.0-SNAPSHOT +3.0.0-SNAPSHOT 0.98.4-hadoop2 - 1.6.1-SNAPSHOT + 1.6.1
[2/9] git commit: SLIDER-441. Rename DO_NOT_PROPAGATE to be more user-friendly
SLIDER-441. Rename DO_NOT_PROPAGATE to be more user-friendly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9ce88ced
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9ce88ced
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9ce88ced
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 9ce88ced4fe24c5284e110877200367d24248430
Parents: 47b4a9c
Author: Sumit Mohanty
Authored: Tue Sep 23 21:34:00 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Sep 23 21:34:00 2014 -0700
--
app-packages/memcached-win/appConfig.json | 2 +-
app-packages/memcached/appConfig.json | 2 +-
app-packages/storm/appConfig.json | 4 ++--
.../src/main/python/agent/CustomServiceOrchestrator.py| 8
.../test/python/agent/TestCustomServiceOrchestrator.py| 10 +-
.../slider/providers/agent/AgentProviderService.java | 4 ++--
.../slider/providers/agent/TestAgentProviderService.java | 2 +-
7 files changed, 16 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/memcached-win/appConfig.json
--
diff --git a/app-packages/memcached-win/appConfig.json
b/app-packages/memcached-win/appConfig.json
index 7863579..059c712 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -13,7 +13,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
-"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/memcached/appConfig.json
--
diff --git a/app-packages/memcached/appConfig.json
b/app-packages/memcached/appConfig.json
index cf4a838..7b4aa97 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -14,7 +14,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
-"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+"site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/storm/appConfig.json
--
diff --git a/app-packages/storm/appConfig.json
b/app-packages/storm/appConfig.json
index 9a926c2..683b8cc 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -33,10 +33,10 @@
"site.storm-site.transactional.zookeeper.port": "null",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
-"site.storm-site.supervisor.slots.ports":
"[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
+"site.storm-site.supervisor.slots.ports":
"[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
"site.storm-site.supervisor.childopts": "-Xmx256m
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.port=0
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
-"site.storm-site.logviewer.port":
"${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+"site.storm-site.logviewer.port":
"${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index dd8e9b9..30eedde 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -246,7 +246,7 @@ class CustomServiceOrchestrat
[5/9] git commit: SLIDER-450 Rename template and resources json files to -default
SLIDER-450 Rename template and resources json files to -default
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5470c47a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5470c47a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5470c47a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 5470c47aae8af30e62404a69cce6fc3e573efa24
Parents: a34837c
Author: tedyu
Authored: Wed Sep 24 15:38:57 2014 -0700
Committer: tedyu
Committed: Wed Sep 24 15:38:57 2014 -0700
--
app-packages/hbase-win/appConfig-default.json | 38
app-packages/hbase-win/appConfig.json | 38
app-packages/hbase-win/resources-default.json | 24 ++
app-packages/hbase-win/resources.json | 24 --
app-packages/hbase-win/src/assembly/hbase.xml | 4 +-
app-packages/hbase/appConfig-default.json | 47
app-packages/hbase/appConfig.json | 47
app-packages/hbase/pom.xml | 4 +-
app-packages/hbase/resources-default.json | 36 +++
app-packages/hbase/resources.json | 36 ---
app-packages/hbase/src/assembly/hbase.xml | 4 +-
.../src/test/resources/resources-default.json | 24 ++
.../hbase/src/test/resources/resources.json | 24 --
13 files changed, 175 insertions(+), 175 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/appConfig-default.json
--
diff --git a/app-packages/hbase-win/appConfig-default.json
b/app-packages/hbase-win/appConfig-default.json
new file mode 100644
index 000..5b97e69
--- /dev/null
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -0,0 +1,38 @@
+{
+"schema": "http://example.org/specification/v2.0.0";,
+"metadata": {
+},
+"global": {
+"application.def":
".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
+"create.default.zookeeper.node": "true",
+"java_home": "C:\\java",
+
+"site.global.app_user": "hadoop",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+"site.global.hbase_instance_name": "instancename",
+"site.global.hbase_root_password": "secret",
+"site.global.user_group": "hadoop",
+"site.global.security_enabled": "false",
+"site.global.monitor_protocol": "http",
+"site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+"site.global.java_library_path":
"c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
+
+"site.hbase-env.hbase_master_heapsize": "1024m",
+"site.hbase-env.hbase_regionserver_heapsize": "1024m",
+"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+"site.hbase-site.hbase.superuser": "yarn",
+"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+"site.hbase-site.hbase.regionserver.info.port": "0",
+"site.hbase-site.hbase.master.info.port":
"${HBASE_MASTER.ALLOCATED_PORT}",
+"site.hbase-site.hbase.regionserver.port": "0",
+"site.hbase-site.hbase.master.port": "0"
+},
+"components": {
+"slider-appmaster": {
+"jvm.heapsize": "256M"
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/appConfig.json
--
diff --git a/app-packages/hbase-win/appConfig.json
b/app-packages/hbase-win/appConfig.json
deleted file mode 100644
index 5b97e69..000
--- a/app-packages/hbase-win/appConfig.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-"schema": "http://example.org/specification/v2.0.0";,
-"metadata": {
-},
-"global": {
-"application.def":
".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
-"create.default.zookeeper.node": "true",
-"java_home": "C:\\java",
-
-"site.global.app_user": "hadoop",
-"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
-"site.global.hbase_instance_name": "instancename",
-"site.global.hbase_root_password": "secret",
-"site.global.user_group": "hadoop",
-"site.global.security_enabled": "false",
-"site.global.monitor_protocol": "http",
-"site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
-
[3/9] git commit: SLIDER-343. Command files have incorrect hostname
SLIDER-343. Command files have incorrect hostname
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/314c3cce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/314c3cce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/314c3cce
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 314c3cce5d52e11e04e3d591f1795d128e9b722a
Parents: 9ce88ce
Author: Sumit Mohanty
Authored: Wed Sep 24 13:43:13 2014 -0700
Committer: Sumit Mohanty
Committed: Wed Sep 24 13:43:13 2014 -0700
--
.../src/main/python/agent/CustomServiceOrchestrator.py| 7 +--
.../test/python/agent/TestCustomServiceOrchestrator.py| 10 ++
.../server/appmaster/web/rest/agent/ExecutionCommand.java | 1 +
3 files changed, 16 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
--
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 30eedde..7b49a67 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -216,8 +216,11 @@ class CustomServiceOrchestrator():
"""
# Perform few modifications to stay compatible with the way in which
# site.pp files are generated by manifestGenerator.py
-public_fqdn = self.public_fqdn
-command['public_hostname'] = public_fqdn
+command['public_hostname'] = self.public_fqdn
+if 'hostname' in command:
+ command['appmaster_hostname'] = command['hostname']
+command['hostname'] = self.public_fqdn
+
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
--
diff --git
a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 4548aac..2fb1611 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -55,6 +55,7 @@ class TestCustomServiceOrchestrator(TestCase):
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
+ 'hostname' : 'host1',
'componentName': 'NAMENODE',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
@@ -80,6 +81,7 @@ class TestCustomServiceOrchestrator(TestCase):
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
+self.assertEquals(command['hostname'], "host1")
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
@@ -87,6 +89,12 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
+
+# Testing side effect of dump_command_to_json
+self.assertEquals(command['public_hostname'], "test.hst")
+self.assertEquals(command['hostname'], "test.hst")
+self.assertEquals(command['appmaster_hostname'], "host1")
+
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
json_file = orchestrator.dump_command_to_json(command, {})
@@ -97,6 +105,8 @@ class TestCustomServiceOrchestrator(TestCase):
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
+self.assertEquals(command['hostname'], "test.hst")
+self.assertEquals(command['appmaster_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index 94aa29f..5fb3b5e 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++
b/slider-core/
[8/9] git commit: SLIDER-459. Environment variable for containers do not expand correctly
SLIDER-459. Environment variable for containers do not expand correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43f9a94a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43f9a94a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43f9a94a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 43f9a94ad364f691069723667983f1b0ed285a55
Parents: 772759c
Author: Sumit Mohanty
Authored: Thu Sep 25 23:55:23 2014 -0700
Committer: Sumit Mohanty
Committed: Thu Sep 25 23:55:23 2014 -0700
--
slider-agent/src/main/python/agent/ProcessHelper.py| 5 +++--
slider-agent/src/main/python/agent/main.py | 13 +
.../org/apache/slider/providers/agent/AgentKeys.java | 2 +-
.../slider/providers/agent/AgentProviderService.java | 2 +-
4 files changed, 14 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-agent/src/main/python/agent/ProcessHelper.py
--
diff --git a/slider-agent/src/main/python/agent/ProcessHelper.py
b/slider-agent/src/main/python/agent/ProcessHelper.py
index 467c4d8..81737f4 100644
--- a/slider-agent/src/main/python/agent/ProcessHelper.py
+++ b/slider-agent/src/main/python/agent/ProcessHelper.py
@@ -24,11 +24,12 @@ import traceback
import sys
import posixpath
from shell import getTempFiles
+import Constants
logger = logging.getLogger()
-if 'AGENT_WORK_ROOT' in os.environ:
- pidfile = os.path.realpath(posixpath.join(os.environ['AGENT_WORK_ROOT'],
"infra", "run", "agent.pid"))
+if Constants.AGENT_WORK_ROOT in os.environ:
+ pidfile =
os.path.realpath(posixpath.join(os.environ[Constants.AGENT_WORK_ROOT], "infra",
"run", "agent.pid"))
else:
pidfile = None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-agent/src/main/python/agent/main.py
--
diff --git a/slider-agent/src/main/python/agent/main.py
b/slider-agent/src/main/python/agent/main.py
index a8fc4e1..48d84cd 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -43,7 +43,7 @@ formatstr = "%(levelname)s %(asctime)s
%(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
configFileRelPath = "infra/conf/agent.ini"
-logFileName = "agent.log"
+logFileName = "slider-agent.log"
SERVER_STATUS_URL="https://{0}:{1}{2}";
@@ -185,9 +185,14 @@ def main():
parser.add_option("--debug", dest="debug", help="Agent debug hint",
default="")
(options, args) = parser.parse_args()
- if not 'AGENT_WORK_ROOT' in os.environ:
-parser.error("AGENT_WORK_ROOT environment variable must be set.")
- options.root_folder = os.environ['AGENT_WORK_ROOT']
+ if not Constants.AGENT_WORK_ROOT in os.environ and not 'PWD' in os.environ:
+parser.error("AGENT_WORK_ROOT environment variable or PWD must be set.")
+ if Constants.AGENT_WORK_ROOT in os.environ:
+options.root_folder = os.environ[Constants.AGENT_WORK_ROOT]
+ else:
+# some launch environments do not end up setting all environment variables
+options.root_folder = os.environ['PWD']
+
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 1da99b0..c271e8e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -91,7 +91,7 @@ public interface AgentKeys {
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
- String AGENT_OUT_FILE = "agent.out";
+ String AGENT_OUT_FILE = "slider-agent.out";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f37150e..647c6a8 1
git commit: SLIDE 149; sync with YARN-913 which has moved to specific exception for NoPathPermissions (as hadoop.fs one didn't let me set text; propagate through the codebase)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry a3e5b372c ->
969b734fc
SLIDE 149; sync with YARN-913 which has moved to specific exception for
NoPathPermissions (as hadoop.fs one didn't let me set text; propagate through
the codebase)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/969b734f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/969b734f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/969b734f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 969b734fc03ac5684cbbcd326d475f44810560c0
Parents: a3e5b37
Author: Steve Loughran
Authored: Sun Sep 28 11:22:47 2014 +0100
Committer: Steve Loughran
Committed: Sun Sep 28 11:22:47 2014 +0100
--
.../appmaster/web/rest/registry/PathEntryResource.java | 8 +---
.../appmaster/web/rest/registry/RegistryResource.java | 12 +++-
.../web/rest/registry/TestRegistryRestResources.groovy | 2 +-
3 files changed, 5 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/969b734f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
index 8790871..91da408 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
@@ -18,16 +18,10 @@
package org.apache.slider.server.appmaster.web.rest.registry;
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import
org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.List;
/**
@@ -38,7 +32,7 @@ import java.util.List;
public class PathEntryResource {
/**
- * Child nodes: as the full path to each element
+ * Child nodes: as the short path to each element
*/
public List nodes;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/969b734f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
index 23f7261..fc41e7d 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -19,22 +19,19 @@
package org.apache.slider.server.appmaster.web.rest.registry;
import com.google.inject.Singleton;
-import org.apache.hadoop.fs.PathAccessDeniedException;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import
org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException;
import
org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import
org.apache.hadoop.yarn.registry.client.exceptions.NoPathPermissionsException;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -42,13 +39,11 @@ import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
-import java.util.List;
/**
- * This is the
[12/12] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/main/java/org/apache/slider/client/SliderClient.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/733745ea Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/733745ea Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/733745ea Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 733745eaf1cf3734dd8f25e7ff0945c253c4777f Parents: f058495 41ec741 Author: Steve Loughran Authored: Tue Sep 30 17:48:02 2014 -0700 Committer: Steve Loughran Committed: Tue Sep 30 17:48:02 2014 -0700 -- app-packages/accumulo/appConfig-default.json| 57 ++ app-packages/accumulo/appConfig.json| 57 -- app-packages/accumulo/pom.xml | 10 + app-packages/accumulo/resources-default.json| 39 ++ app-packages/accumulo/resources.json| 39 -- app-packages/accumulo/src/assembly/accumulo.xml | 4 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 3 +- app-packages/hbase-win/README.txt | 3 - app-packages/hbase-win/appConfig-default.json | 2 +- app-packages/hbase/appConfig-default.json | 2 +- app-packages/hbase/resources-default.json | 3 + app-packages/storm-win/README.txt | 36 ++ app-packages/storm-win/appConfig-default.json | 41 ++ .../storm-win/configuration/storm-env.xml | 65 +++ .../storm-win/configuration/storm-site.xml | 580 +++ app-packages/storm-win/metainfo.xml | 149 + .../storm-win/package/scripts/drpc_server.py| 55 ++ .../storm-win/package/scripts/nimbus.py | 55 ++ .../storm-win/package/scripts/params.py | 56 ++ .../storm-win/package/scripts/rest_api.py | 57 ++ .../storm-win/package/scripts/service.py| 56 ++ .../storm-win/package/scripts/status_params.py | 37 ++ app-packages/storm-win/package/scripts/storm.py | 53 ++ .../storm-win/package/scripts/supervisor.py | 61 ++ .../storm-win/package/scripts/ui_server.py | 55 ++ .../storm-win/package/scripts/yaml_config.py| 80 +++ .../storm-win/package/templates/config.yaml.j2 | 28 + .../package/templates/storm_jaas.conf.j2| 44 ++ app-packages/storm-win/pom.xml | 91 +++ app-packages/storm-win/resources-default.json | 30 + app-packages/storm-win/src/assembly/storm.xml | 68 +++ app-packages/storm/appConfig-default.json | 2 +- slider-agent/src/main/python/kazoo/client.py| 56 +- .../src/main/python/kazoo/handlers/utils.py | 42 +- .../main/python/kazoo/protocol/connection.py| 74 +-- .../src/main/python/kazoo/tests/test_client.py | 55 +- .../main/python/kazoo/tests/test_connection.py | 89 ++- slider-core/pom.xml | 6 + .../org/apache/slider/client/SliderClient.java | 291 +- .../common/SliderXMLConfKeysForTesting.java | 1 + .../common/params/ActionDiagnosticArgs.java | 66 +++ .../apache/slider/common/params/Arguments.java | 7 + .../apache/slider/common/params/ClientArgs.java | 11 +- .../slider/common/params/SliderActions.java | 4 + .../apache/slider/common/tools/SliderUtils.java | 108 .../slider/providers/agent/AgentKeys.java | 5 + .../providers/agent/AgentProviderService.java | 2 +- .../services/security/CertificateManager.java | 48 +- .../providers/slideram/instance/appconf.json| 3 +- .../agent/TestAgentAMManagementWS.groovy| 131 +++-- .../apache/slider/test/SliderTestUtils.groovy | 3 +- .../framework/AgentCommandTestBase.groovy | 7 + .../funtest/lifecycle/AMFailuresIT.groovy | 6 + .../clusters/remote/slider/slider-client.xml| 6 + 54 files changed, 2595 insertions(+), 344 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 99896c8,06c37ba..5d994f1 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@@ -133,8 -136,8
[06/12] git commit: SLIDER-450 additional fix to use default appConfig for accumulo testing
SLIDER-450 additional fix to use default appConfig for accumulo testing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2a3a4ed6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2a3a4ed6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2a3a4ed6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2a3a4ed67789aa0833fe00ca1bb5bb9bd80816d1
Parents: f8210a7
Author: Billie Rinaldi
Authored: Sat Sep 27 13:02:51 2014 -0700
Committer: Billie Rinaldi
Committed: Sat Sep 27 13:02:51 2014 -0700
--
app-packages/accumulo/pom.xml| 8
.../apache/slider/funtest/accumulo/AccumuloBasicIT.groovy| 4 +---
2 files changed, 9 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a3a4ed6/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index a2014f2..bc4b591 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -210,6 +210,14 @@
true
${test.app.resources.dir}
+
+.
+true
+${test.app.resources.dir}
+
+ appConfig-default.json
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a3a4ed6/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index dad7601..12c0655 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -56,9 +56,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
String appTemplateFile = templateName()
Configuration conf = new Configuration()
FileSystem fs = FileSystem.getLocal(conf)
-InputStream stream = SliderUtils.getApplicationResourceInputStream(
- fs, new Path(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE),
- "appConfig-default.json");
+InputStream stream = new FileInputStream(sysprop("test.app.resources.dir")
+ "/appConfig-default.json")
assert stream!=null, "Couldn't pull appConfig.json from app pkg"
ConfTreeSerDeser c = new ConfTreeSerDeser()
ConfTree t = c.fromStream(stream)
[05/12] git commit: SLIDER-158. Add a slider diagnostics command (Thomas Liu via smohanty)
SLIDER-158. Add a slider diagnostics command (Thomas Liu via smohanty) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f8210a7c Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f8210a7c Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f8210a7c Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: f8210a7c00c54054dc690abbe5088cd4ddea4e3e Parents: ce647d3 Author: Sumit Mohanty Authored: Sat Sep 27 00:22:12 2014 -0700 Committer: Sumit Mohanty Committed: Sat Sep 27 00:22:26 2014 -0700 -- slider-core/pom.xml | 6 + .../org/apache/slider/client/SliderClient.java | 280 ++- .../common/params/ActionDiagnosticArgs.java | 66 + .../apache/slider/common/params/Arguments.java | 7 + .../apache/slider/common/params/ClientArgs.java | 11 +- .../slider/common/params/SliderActions.java | 4 + .../apache/slider/common/tools/SliderUtils.java | 108 +++ .../slider/providers/agent/AgentKeys.java | 5 + 8 files changed, 485 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/pom.xml -- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 39c17c6..7af8bd7 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -448,6 +448,12 @@ + org.codehaus.jettison + jettison + 1.1 + + + org.powermock powermock-core 1.5 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/client/SliderClient.java -- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index d57f7dc..06c37ba 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -19,6 +19,7 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -33,6 +34,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -48,6 +51,7 @@ import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionCreateArgs; @@ -91,6 +95,7 @@ import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.RunningApplication; import org.apache.slider.core.main.RunService; import org.apache.slider.core.persist.ConfPersister; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.registry.YARNRegistryClient; import org.apache.slider.core.registry.docstore.ConfigFormat; @@ -115,9 +120,15 @@ import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.FileWriter; @@ -125,6 +136,7 @@ import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; impor
[08/12] git commit: Add log aggregation entries to resources-default.json
Add log aggregation entries to resources-default.json
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f392d23e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f392d23e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f392d23e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f392d23ee9342a8e250132ce9ad093d19a4ae234
Parents: ba1e79b
Author: tedyu
Authored: Mon Sep 29 19:58:49 2014 -0700
Committer: tedyu
Committed: Mon Sep 29 19:58:49 2014 -0700
--
app-packages/hbase/resources-default.json | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f392d23e/app-packages/hbase/resources-default.json
--
diff --git a/app-packages/hbase/resources-default.json
b/app-packages/hbase/resources-default.json
index c3fec68..2487e14 100644
--- a/app-packages/hbase/resources-default.json
+++ b/app-packages/hbase/resources-default.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+"yarn.log.include.patterns": "",
+"yarn.log.exclude.patterns": "",
+"yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
[01/12] git commit: SLIDER-450 rename sample template and resources files for accumulo
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 969b734fc ->
733745eaf
SLIDER-450 rename sample template and resources files for accumulo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bb4e7d31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bb4e7d31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bb4e7d31
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bb4e7d31704d8d711ff85288166192f7d41d2648
Parents: 43f9a94
Author: Billie Rinaldi
Authored: Fri Sep 26 07:29:34 2014 -0700
Committer: Billie Rinaldi
Committed: Fri Sep 26 07:29:34 2014 -0700
--
app-packages/accumulo/appConfig-default.json| 57
app-packages/accumulo/appConfig.json| 57
app-packages/accumulo/pom.xml | 2 +
app-packages/accumulo/resources-default.json| 39 ++
app-packages/accumulo/resources.json| 39 --
app-packages/accumulo/src/assembly/accumulo.xml | 4 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +-
7 files changed, 102 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/appConfig-default.json
--
diff --git a/app-packages/accumulo/appConfig-default.json
b/app-packages/accumulo/appConfig-default.json
new file mode 100644
index 000..62050af
--- /dev/null
+++ b/app-packages/accumulo/appConfig-default.json
@@ -0,0 +1,57 @@
+{
+ "schema": "http://example.org/specification/v2.0.0";,
+ "metadata": {
+ },
+ "global": {
+"application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
+"java_home": "/usr/lib/jvm/java",
+"site.global.app_user": "${app.user}",
+"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
+"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+"site.global.tserver_heapsize": "256m",
+"site.global.master_heapsize": "128m",
+"site.global.monitor_heapsize": "64m",
+"site.global.gc_heapsize": "64m",
+"site.global.other_heapsize": "128m",
+"site.global.hadoop_prefix": "${hadoop.dir}/hadoop",
+"site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+"site.global.zookeeper_home": "${zk.dir}/zookeeper",
+"site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
+"site.global.accumulo_root_password": "NOT_USED",
+"site.global.user_group": "${app.user.group}",
+"site.global.security_enabled": "false",
+"site.global.ssl_cert_dir": "ssl",
+"site.global.monitor_protocol": "http",
+"site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+"site.accumulo-site.instance.security.authenticator":
"org.apache.slider.accumulo.CustomAuthenticator",
+"site.accumulo-site.general.security.credential.provider.paths":
"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+"site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}",
+"site.accumulo-site.general.kerberos.principal": "${accumulo.principal}",
+"site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+"site.accumulo-site.tserver.memory.maps.max": "80M",
+"site.accumulo-site.tserver.cache.data.size": "7M",
+"site.accumulo-site.tserver.cache.index.size": "20M",
+"site.accumulo-site.trace.user": "root",
+"site.accumulo-site.tserver.sort.buffer.size": "50M",
+"site.accumulo-site.tserver.walog.max.size": "40M",
+"site.accumulo-site.master.port.client": "0",
+"site.accumulo-site.trace.port.client": "0",
+"site.accumulo-site.tserver.port.client": "0",
+"site.accumulo-site.gc.port.client": "0",
+"site.accumulo-site.monitor.port.client":
"${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+"site.accumulo-site.monitor.port.log4j": "0",
+"site.accumulo-site.master.replication.coordinator.port": "0",
+"site.accumulo-site.replication.receipt.service.port": "0",
+"site.accumulo-site.general.classpaths":
"$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$H
[10/12] git commit: SLIDER-149 sync with move to short-named status
SLIDER-149 sync with move to short-named status
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/62335bfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/62335bfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/62335bfd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 62335bfd1a894db41016fb0aa14268c5e21737e2
Parents: 969b734
Author: Steve Loughran
Authored: Tue Sep 30 15:24:18 2014 -0700
Committer: Steve Loughran
Committed: Tue Sep 30 15:24:18 2014 -0700
--
.../TestStandaloneYarnRegistryAM.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 7 +
.../minicluster/live/TestTwoLiveClusters.groovy | 32
3 files changed, 28 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index d081676..0155232 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -154,6 +154,7 @@ class TestStandaloneYarnRegistryAM extends
AgentMiniClusterTestBase {
Map < String, ServiceRecord > recordMap =
RecordOperations.extractServiceRecords(
registryService,
+recordsPath,
registryService.listFull(recordsPath))
def serviceRecords = recordMap.values();
dumpCollection(serviceRecords)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 9f8e850..fe9e587 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -713,6 +713,13 @@ class SliderTestUtils extends Assert {
log.info("number of entries: ${entries.length}")
entries.each { log.info(it.toString()) }
}
+
+ public static void dumpMap(Map map) {
+map.entrySet().each { Map.Entry it ->
+ log.info("\"${it.key.toString()}\": \"${it.value.toString()}\"")
+}
+ }
+
/**
* Get a time option in seconds if set, otherwise the default value (also in
seconds).
* This operation picks up the time value as a system property if set -that
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
--
diff --git
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index a415524..01a67c7 100644
---
a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++
b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -20,11 +20,11 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.HBaseKeys
@@ -87,16 +87,20 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
describe "service registry names"
RegistryOperations registry = cluster2Client.registryOperations
def home = RegistryOperationUtils.homePathForCur
[11/12] git commit: SLIDER149 in sync with YARN-913 changes
SLIDER149 in sync with YARN-913 changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f058495a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f058495a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f058495a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f058495a6c05a385518333d135b84d357780c7f6
Parents: 62335bf
Author: Steve Loughran
Authored: Tue Sep 30 17:42:22 2014 -0700
Committer: Steve Loughran
Committed: Tue Sep 30 17:42:22 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 6 +++---
.../YarnRegistryViewForProviders.java | 10 -
.../TestStandaloneYarnRegistryAM.groovy | 17 ++-
.../registry/TestRegistryRestResources.groovy | 4 ++--
.../minicluster/live/TestHBaseMaster.groovy | 8 +++
.../minicluster/live/TestTwoLiveClusters.groovy | 22 +---
6 files changed, 30 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 88d07b5..99896c8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import static
org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*;
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
@@ -1455,7 +1455,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
public String getUsername() throws IOException {
-return RegistryOperationUtils.currentUser();
+return RegistryUtils.currentUser();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index e5b150a..d0c891d 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -20,7 +20,7 @@ package org.apache.slider.server.services.yarnregistry;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.api.CreateFlags;
@@ -104,7 +104,7 @@ public class YarnRegistryViewForProviders {
String serviceName,
String componentName,
ServiceRecord record) throws IOException {
-String path = RegistryOperationUtils.componentPath(
+String path = RegistryUtils.componentPath(
user, serviceClass, serviceName, componentName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
@@ -122,7 +122,7 @@ public class YarnRegistryViewForProviders {
String serviceClass,
String serviceName,
ServiceRecord record) throws IOException {
-String path = RegistryOperationUtils.servicePath(
+String path = RegistryUtils.servicePath(
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
@@ -139,1
[02/12] git commit: SLIDER-461 Asymmetry in default arguments
SLIDER-461 Asymmetry in default arguments
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/964e4033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/964e4033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/964e4033
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 964e4033d5d0df4c6f15e016417af664f0f8ac31
Parents: bb4e7d3
Author: tedyu
Authored: Fri Sep 26 09:14:48 2014 -0700
Committer: tedyu
Committed: Fri Sep 26 09:14:48 2014 -0700
--
app-packages/hbase-win/appConfig-default.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/storm/appConfig-default.json | 2 +-
.../org/apache/slider/providers/agent/AgentProviderService.java| 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/hbase-win/appConfig-default.json
--
diff --git a/app-packages/hbase-win/appConfig-default.json
b/app-packages/hbase-win/appConfig-default.json
index 5b97e69..95e79ab 100644
--- a/app-packages/hbase-win/appConfig-default.json
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -24,7 +24,7 @@
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
-"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+"site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
"site.hbase-site.hbase.master.info.port":
"${HBASE_MASTER.ALLOCATED_PORT}",
"site.hbase-site.hbase.regionserver.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/hbase/appConfig-default.json
--
diff --git a/app-packages/hbase/appConfig-default.json
b/app-packages/hbase/appConfig-default.json
index 78479db..c7952df 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -33,7 +33,7 @@
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
-"site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+"site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
"site.hbase-site.hbase.master.info.port":
"${HBASE_MASTER.ALLOCATED_PORT}",
"site.hbase-site.hbase.regionserver.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/storm/appConfig-default.json
--
diff --git a/app-packages/storm/appConfig-default.json
b/app-packages/storm/appConfig-default.json
index 683b8cc..2b0eea0 100644
--- a/app-packages/storm/appConfig-default.json
+++ b/app-packages/storm/appConfig-default.json
@@ -29,7 +29,7 @@
"site.storm-site.worker.childopts": "-Xmx768m
-javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path":
"${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
-"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
+"site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}",
"site.storm-site.transactional.zookeeper.port": "null",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 647c6a8..755b9d8 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/provi
[03/12] git commit: SLIDER-443 remove default keystore location SLIDER-449 Consume both streams for openssl (required for windows)
SLIDER-443 remove default keystore location
SLIDER-449 Consume both streams for openssl (required for windows)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/732569db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/732569db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/732569db
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 732569db9468ed7ced6fe5ae750130fdcd455d04
Parents: 964e403
Author: Jon Maron
Authored: Fri Sep 26 17:07:18 2014 -0400
Committer: Jon Maron
Committed: Fri Sep 26 17:07:18 2014 -0400
--
.../org/apache/slider/client/SliderClient.java | 12 +-
.../services/security/CertificateManager.java | 48 ++-
.../providers/slideram/instance/appconf.json| 3 +-
.../agent/TestAgentAMManagementWS.groovy| 131 ---
.../apache/slider/test/SliderTestUtils.groovy | 3 +-
5 files changed, 141 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index afbb4a8..d57f7dc 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -782,7 +782,8 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
builder.init(providerName, instanceDefinition);
builder.propagateFilename();
builder.propagatePrincipals();
-builder.setImageDetailsIfAvailable(buildInfo.getImage(),
buildInfo.getAppHomeDir());
+builder.setImageDetailsIfAvailable(buildInfo.getImage(),
+ buildInfo.getAppHomeDir());
builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
@@ -839,7 +840,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
throw e;
}
try {
- builder.persist(appconfdir, overwrite);
+ persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on {} : {}", builder, e);
throw new BadClusterStateException("Failed to save " + clustername
@@ -847,6 +848,13 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
}
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+builder.persist(appconfdir, overwrite);
+ }
+
@VisibleForTesting
public static void replaceTokens(ConfTree conf,
String userName, String clusterName) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 8f679e1..d200033 100644
---
a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++
b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.text.MessageFormat;
@@ -86,6 +87,42 @@ public class CertificateManager {
return certFile.exists();
}
+ class StreamConsumer extends Thread
+ {
+InputStream is;
+boolean logOutput;
+
+StreamConsumer(InputStream is, boolean logOutput)
+{
+ this.is = is;
+ this.logOutput = logOutput;
+}
+
+StreamConsumer(InputStream is)
+{
+ this(is, false);
+}
+
+public void run()
+{
+ try
+ {
+InputStreamReader isr = new InputStreamReader(is,
+ Charset.forName("UTF8"));
+BufferedReader br = new BufferedReader(isr);
+String line;
+while ( (line = br.readLine()) != null)
+ if (logOutput) {
+LOG.info(line);
+ }
+ } catch (IOException e)
[04/12] git commit: SLIDER-389. Upgrade kazoo code base to the latest version
SLIDER-389. Upgrade kazoo code base to the latest version
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce647d39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce647d39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce647d39
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ce647d399acf5b2d90e51a8509633183438e11d3
Parents: 732569d
Author: Sumit Mohanty
Authored: Fri Sep 26 14:20:49 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Sep 26 14:20:49 2014 -0700
--
slider-agent/src/main/python/kazoo/client.py| 56 +++-
.../src/main/python/kazoo/handlers/utils.py | 42 +
.../main/python/kazoo/protocol/connection.py| 74
.../src/main/python/kazoo/tests/test_client.py | 55 ++--
.../main/python/kazoo/tests/test_connection.py | 89
5 files changed, 137 insertions(+), 179 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/client.py
--
diff --git a/slider-agent/src/main/python/kazoo/client.py
b/slider-agent/src/main/python/kazoo/client.py
index 11d9008..47545ee 100644
--- a/slider-agent/src/main/python/kazoo/client.py
+++ b/slider-agent/src/main/python/kazoo/client.py
@@ -20,7 +20,7 @@ from kazoo.exceptions import (
WriterNotClosedException,
)
from kazoo.handlers.threading import SequentialThreadingHandler
-from kazoo.handlers.utils import capture_exceptions, wrap, pipe_or_sock_write
+from kazoo.handlers.utils import capture_exceptions, wrap
from kazoo.hosts import collect_hosts
from kazoo.loggingsupport import BLATHER
from kazoo.protocol.connection import ConnectionHandler
@@ -220,7 +220,6 @@ class KazooClient(object):
elif type(command_retry) is KazooRetry:
self.retry = command_retry
-
if type(self._conn_retry) is KazooRetry:
if self.handler.sleep_func != self._conn_retry.sleep_func:
raise ConfigurationError("Retry handler and event handler "
@@ -228,19 +227,21 @@ class KazooClient(object):
if type(self.retry) is KazooRetry:
if self.handler.sleep_func != self.retry.sleep_func:
-raise ConfigurationError("Command retry handler and event "
- "handler must use the same sleep
func")
+raise ConfigurationError(
+"Command retry handler and event handler "
+"must use the same sleep func")
if self.retry is None or self._conn_retry is None:
old_retry_keys = dict(_RETRY_COMPAT_DEFAULTS)
for key in old_retry_keys:
try:
old_retry_keys[key] = kwargs.pop(key)
-warnings.warn('Passing retry configuration param %s to the'
-' client directly is deprecated, please pass a'
-' configured retry object (using param %s)' % (
-key, _RETRY_COMPAT_MAPPING[key]),
-DeprecationWarning, stacklevel=2)
+warnings.warn(
+'Passing retry configuration param %s to the '
+'client directly is deprecated, please pass a '
+'configured retry object (using param %s)' % (
+key, _RETRY_COMPAT_MAPPING[key]),
+DeprecationWarning, stacklevel=2)
except KeyError:
pass
@@ -258,12 +259,13 @@ class KazooClient(object):
**retry_keys)
self._conn_retry.interrupt = lambda: self._stopped.is_set()
-self._connection = ConnectionHandler(self, self._conn_retry.copy(),
-logger=self.logger)
+self._connection = ConnectionHandler(
+self, self._conn_retry.copy(), logger=self.logger)
# Every retry call should have its own copy of the retry helper
# to avoid shared retry counts
self._retry = self.retry
+
def _retry(*args, **kwargs):
return self._retry.copy()(*args, **kwargs)
self.retry = _retry
@@ -282,7 +284,7 @@ class KazooClient(object):
self.Semaphore = partial(Semaphore, self)
self.ShallowParty = partial(ShallowParty, self)
- # If we got any unhandled keywords, complain like python would
+# If we got any unhandled keywords, complain like Python would
if kwargs:
raise TypeError('__init__() got unexpected keyword arguments: %s'
% (kwargs.keys(),))
@@ -433,7 +43
[09/12] git commit: SLIDER-180. Add storm package for windows (I)
SLIDER-180. Add storm package for windows (I)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41ec7410
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41ec7410
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41ec7410
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 41ec7410f047e28c114a77f67ac6f97763b139ad
Parents: f392d23
Author: Sumit Mohanty
Authored: Tue Sep 30 09:05:31 2014 -0700
Committer: Sumit Mohanty
Committed: Tue Sep 30 09:05:31 2014 -0700
--
app-packages/hbase-win/README.txt | 3 -
app-packages/storm-win/README.txt | 36 ++
app-packages/storm-win/appConfig-default.json | 41 ++
.../storm-win/configuration/storm-env.xml | 65 +++
.../storm-win/configuration/storm-site.xml | 580 +++
app-packages/storm-win/metainfo.xml | 149 +
.../storm-win/package/scripts/drpc_server.py| 55 ++
.../storm-win/package/scripts/nimbus.py | 55 ++
.../storm-win/package/scripts/params.py | 56 ++
.../storm-win/package/scripts/rest_api.py | 57 ++
.../storm-win/package/scripts/service.py| 56 ++
.../storm-win/package/scripts/status_params.py | 37 ++
app-packages/storm-win/package/scripts/storm.py | 53 ++
.../storm-win/package/scripts/supervisor.py | 61 ++
.../storm-win/package/scripts/ui_server.py | 55 ++
.../storm-win/package/scripts/yaml_config.py| 80 +++
.../storm-win/package/templates/config.yaml.j2 | 28 +
.../package/templates/storm_jaas.conf.j2| 44 ++
app-packages/storm-win/pom.xml | 91 +++
app-packages/storm-win/resources-default.json | 30 +
app-packages/storm-win/src/assembly/storm.xml | 68 +++
21 files changed, 1697 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/hbase-win/README.txt
--
diff --git a/app-packages/hbase-win/README.txt
b/app-packages/hbase-win/README.txt
index 0e64a84..4e0e30a 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -22,9 +22,6 @@ These files are included as reference configuration for
Slider apps and are suit
for a one-node cluster.
-OPTION-I: Use a downloaded hbase tarball fro Windows
-
-** OPTION - I **
To create the app package you will need the HBase tarball and invoke mvn
command
with appropriate parameters.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/README.txt
--
diff --git a/app-packages/storm-win/README.txt
b/app-packages/storm-win/README.txt
new file mode 100644
index 000..8631714
--- /dev/null
+++ b/app-packages/storm-win/README.txt
@@ -0,0 +1,36 @@
+
+
+How to create a Slider app package for Storm?
+
+To create the app package you will need the Storm tarball and invoke mvn
command
+with appropriate parameters.
+
+Command:
+mvn clean package -Pstorm-app-package-win -Dpkg.version=
+ -Dpkg.name= -Dpkg.src=
+
+Example:
+mvn clean package -Pstorm-app-package-win -Dpkg.version=0.9.3
+ -Dpkg.name=storm-0.9.3.zip -Dpkg.src=/Users/user1/Downloads
+
+App package can be found in
+ app-packages/storm-win/target/slider-storm-app-win-package-${pkg.version}.zip
+
+appConfig-default.json and resources-default.json are not required to be
packaged.
+These files are included as reference configuration for Slider apps and are
suitable
+for a one-node cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/appConfig-default.json
--
diff --git a/app-packages/storm-win/appConfig-default.json
b/app-packages/storm-win/appConfig-default.json
new file mode 100644
index 000..428dea5
--- /dev/null
+++ b/app-packages/storm-win/appConfig-default.json
@@ -0,0 +1,41 @@
+{
+ "schema": "http://example.org/specification/v2.0.0";,
+ "metadata": {
+ },
+ "global": {
+"application.def":
".slider/package/STORM/slider-storm-app-win-package-${pkg.version}.zip",
+"java_home": "C:\\java",
+"create.default.zookeeper.node": "true",
+
+"site.global.app_user": "hadoop",
+"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/storm-${pkg.version}",
+"site.global.user_group": "hadoop",
+"site.global.security_enabled": "false",
+"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
+"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
+
+"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
+"site.storm-site.storm.zookeeper.servers":
[07/12] git commit: SLIDER-448 AMFailuresIT needs vagrant
SLIDER-448 AMFailuresIT needs vagrant
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ba1e79b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ba1e79b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ba1e79b9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ba1e79b9e0447bdf5b5173b7d4d17477e33fad19
Parents: 2a3a4ed
Author: Gour Saha
Authored: Mon Sep 29 16:55:54 2014 -0700
Committer: Gour Saha
Committed: Mon Sep 29 16:55:54 2014 -0700
--
.../org/apache/slider/common/SliderXMLConfKeysForTesting.java | 1 +
.../slider/funtest/framework/AgentCommandTestBase.groovy | 7 +++
.../org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy | 6 ++
src/test/clusters/remote/slider/slider-client.xml | 6 ++
4 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ba1e79b9/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 4c56240..8886cb7 100644
---
a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++
b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -62,6 +62,7 @@ public interface SliderXMLConfKeysForTesting {
String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
String KEY_AGENTTESTS_QUEUE_LABELED_DEFINED =
"slider.test.agent.labeled.queue.enabled";
String KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED =
"slider.test.agent.labels.defined";
+ String KEY_AGENTTESTS_AM_FAILURES_ENABLED =
"slider.test.agent.am.failures.enabled";
int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ba1e79b9/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
--
diff --git
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index cf32e94..602fe2c 100644
---
a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++
b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -39,6 +39,7 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
public static final boolean AGENTTESTS_ENABLED
public static final boolean AGENTTESTS_QUEUE_LABELED_DEFINED
public static final boolean AGENTTESTS_LABELS_RED_BLUE_DEFINED
+ public static final boolean AGENTTESTS_AM_FAILURES_ENABLED
private static String TEST_APP_PKG_DIR_PROP = "test.app.pkg.dir"
private static String TEST_APP_PKG_FILE_PROP = "test.app.pkg.file"
private static String TEST_APP_PKG_NAME_PROP = "test.app.pkg.name"
@@ -65,6 +66,8 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_QUEUE_LABELED_DEFINED, false)
AGENTTESTS_LABELS_RED_BLUE_DEFINED =
SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED, false)
+AGENTTESTS_AM_FAILURES_ENABLED =
+SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_AM_FAILURES_ENABLED, false)
}
protected String getAppResource() {
@@ -90,6 +93,10 @@ implements FuntestProperties, Arguments, SliderExitCodes,
SliderActions {
assume(AGENTTESTS_LABELS_RED_BLUE_DEFINED, "Custom node labels not
defined")
}
+ public static void assumeAmFailureTestsEnabled() {
+assume(AGENTTESTS_AM_FAILURES_ENABLED, "AM failure tests are disabled")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ba1e79b9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
--
diff --git
a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 852121f..9db0fec 100644
---
a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++
b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -33,6 +33,7 @@ import
org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framew
[1/2] git commit: SLIDER-455 TestStandaloneAMMonkeyRestart test failure
Repository: incubator-slider
Updated Branches:
refs/heads/develop 2bbaafd5d -> 6b011d925
SLIDER-455 TestStandaloneAMMonkeyRestart test failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b1ff8e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b1ff8e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b1ff8e1
Branch: refs/heads/develop
Commit: 0b1ff8e142218a60843697681a7d1419a324f7f8
Parents: 2bbaafd
Author: Steve Loughran
Authored: Wed Oct 1 14:50:43 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 1 14:50:43 2014 -0700
--
.../standalone/TestStandaloneAMMonkeyRestart.groovy | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b1ff8e1/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
index b37b9a6..5136092 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -59,13 +59,11 @@ class TestStandaloneAMMonkeyRestart extends
AgentMiniClusterTestBase {
ApplicationReport report
report = waitForClusterLive(sliderClient, 3)
describe "Waiting for the cluster to fail"
-sleep(4)
-// end of process
-report = sliderClient.applicationReport
-log.info(report.diagnostics)
-assert report.currentApplicationAttemptId.attemptId == threshold
-assert YarnApplicationState.FAILED == report.yarnApplicationState
-assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+def finishedReport = waitForAppToFinish(sliderClient, 9)
+log.info(finishedReport.diagnostics)
+assert finishedReport.currentApplicationAttemptId.attemptId == threshold
+assert YarnApplicationState.FAILED == finishedReport.yarnApplicationState
+assert FinalApplicationStatus.FAILED ==
finishedReport.finalApplicationStatus
}
/**
[2/2] git commit: ide-tuning
ide-tuning
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6b011d92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6b011d92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6b011d92
Branch: refs/heads/develop
Commit: 6b011d92500e6460ac0401ccdd522a84824ea304
Parents: 0b1ff8e
Author: Steve Loughran
Authored: Wed Oct 1 15:40:13 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 1 15:40:13 2014 -0700
--
.../apache/slider/providers/agent/AgentProviderService.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6b011d92/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 755b9d8..d7943b2 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -450,7 +450,7 @@ public class AgentProviderService extends
AbstractProviderService implements
}
Map folders = registration.getLogFolders();
- if (folders != null && folders.size() > 0) {
+ if (folders != null && !folders.isEmpty()) {
publishLogFolderPaths(folders, containerId, roleName, hostFqdn);
}
} else {
@@ -465,9 +465,9 @@ public class AgentProviderService extends
AbstractProviderService implements
/**
* Handle heartbeat response from agents
*
- * @param heartBeat
+ * @param heartBeat incoming heartbeat from Agent
*
- * @return
+ * @return response to send back
*/
@Override
public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
[1/7] git commit: SLIDER-470 slider appears to support negative component counts
Repository: incubator-slider
Updated Branches:
refs/heads/develop 6b011d925 -> 2051dd641
SLIDER-470 slider appears to support negative component counts
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ca99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ca99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ca99
Branch: refs/heads/develop
Commit: ca992abfcd17f9e0fc4f0c677fdbdb0c
Parents: 6b011d9
Author: Steve Loughran
Authored: Thu Oct 2 09:32:52 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 09:32:52 2014 -0700
--
...ndaloneClusterNegativeComponentCounts.groovy | 79
1 file changed, 79 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca99/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
new file mode 100644
index 000..8a23dc5
--- /dev/null
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestStandaloneClusterNegativeComponentCounts
+extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testStandaloneClusterNegativeComponentCounts() throws Throwable {
+describe "Run a Standalone AM with cluster counts set to 0 ... expect
failures"
+// patch the configuration for AM restart
+int threshold = 2;
+YarnConfiguration conf = getRestartableConfiguration(threshold)
+
+String clustername = createMiniCluster("", conf, 1, true)
+if (hdfsCluster) {
+ fail("Agent tests do not (currently) work with mini HDFS cluster")
+}
+ServiceLauncher launcher =
+createCluster(clustername,
+["negative":-4],
+[
+],
+true,
+false,
+agentDefOptions)
+SliderClient sliderClient = launcher.service
+addToTeardown(sliderClient);
+
+ApplicationReport report
+report = waitForClusterLive(sliderClient, 3)
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+def conf = new YarnConfiguration(configuration)
+conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+conf
+ }
+
+
+}
[7/7] git commit: Merge branch 'feature/SLIDER-470_slider_appears_to_support_negative_component_counts' into develop
Merge branch 'feature/SLIDER-470_slider_appears_to_support_negative_component_counts' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2051dd64 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2051dd64 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2051dd64 Branch: refs/heads/develop Commit: 2051dd6411b47c2ec3ea6de0cf3b77ddc054df22 Parents: 6b011d9 cbf3a63 Author: Steve Loughran Authored: Thu Oct 2 14:59:14 2014 -0700 Committer: Steve Loughran Committed: Thu Oct 2 14:59:14 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 48 +- .../apache/slider/core/conf/AggregateConf.java | 20 + .../apache/slider/core/conf/MapOperations.java | 16 +++- .../providers/AbstractClientProvider.java | 9 +- .../apache/slider/providers/ProviderCore.java | 9 +- .../slideram/SliderAMClientProvider.java| 28 ++ .../server/appmaster/SliderAppMaster.java | 52 +++ .../appmaster/actions/ActionStopQueue.java | 8 +- .../server/appmaster/actions/AsyncAction.java | 2 +- .../server/appmaster/actions/QueueExecutor.java | 2 + .../slider/server/appmaster/state/AppState.java | 5 +- .../server/appmaster/state/RoleStatus.java | 20 + .../TestFreezeThawFlexStandaloneAM.groovy | 94 .../TestFreezeThawMasterlessAM.groovy | 92 --- .../standalone/TestBuildStandaloneAM.groovy | 25 -- .../standalone/TestStandaloneAgentAM.groovy | 2 +- .../slider/providers/agent/TestAgentEcho.groovy | 18 +++- .../build/TestBuildThawClusterM1W1.groovy | 3 +- 18 files changed, 298 insertions(+), 155 deletions(-) --
[5/7] git commit: SLIDER-476 make AM queue executor non-reentrant
SLIDER-476 make AM queue executor non-reentrant
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/483ab958
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/483ab958
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/483ab958
Branch: refs/heads/develop
Commit: 483ab958956f52177205ea928730e4bf71935b34
Parents: 755bb7c
Author: Steve Loughran
Authored: Thu Oct 2 14:25:36 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:25:36 2014 -0700
--
.../java/org/apache/slider/server/appmaster/SliderAppMaster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/483ab958/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index cc4f6fe..74a6f34 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -409,7 +409,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newCachedThreadPool(
+Executors.newFixedThreadPool(2,
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
[2/7] git commit: SLIDER-470 slider appears to support negative component counts: client side checks
SLIDER-470 slider appears to support negative component counts: client side
checks
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1deafee8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1deafee8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1deafee8
Branch: refs/heads/develop
Commit: 1deafee8401922f7609ce6333d38b3e52a43f06f
Parents: ca9
Author: Steve Loughran
Authored: Thu Oct 2 13:25:08 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 13:25:08 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 38 ++--
.../apache/slider/core/conf/MapOperations.java | 16 +++--
.../providers/AbstractClientProvider.java | 9 ++---
.../apache/slider/providers/ProviderCore.java | 5 +++
.../slideram/SliderAMClientProvider.java| 28 +++
.../slider/server/appmaster/state/AppState.java | 5 ++-
.../server/appmaster/state/RoleStatus.java | 20 +++
.../standalone/TestBuildStandaloneAM.groovy | 25 +
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
9 files changed, 120 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1deafee8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 06c37ba..05ef532 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -842,17 +842,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// make any substitutions needed at this stage
replaceTokens(appConf.getConfTree(), getUsername(), clustername);
-// provider to validate what there is
-try {
- sliderAM.validateInstanceDefinition(builder.getInstanceDescription());
- provider.validateInstanceDefinition(builder.getInstanceDescription());
-} catch (SliderException e) {
- //problem, reject it
- log.info("Error {} validating application instance definition ",
e.toString());
- log.debug("Error validating application instance definition ", e);
- log.info(instanceDefinition.toString());
- throw e;
-}
+// providers to validate what there is
+AggregateConf instanceDescription = builder.getInstanceDescription();
+validateInstanceDefinition(sliderAM, instanceDescription);
+validateInstanceDefinition(provider, instanceDescription);
try {
persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
@@ -2060,6 +2053,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
clusterDirectory);
+SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+// provider to validate what there is
+SliderAMClientProvider provider = sliderAM;
+validateInstanceDefinition(provider, instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
@@ -2107,6 +2104,25 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+ /**
+ * Validate an instance definition against a provider.
+ * @param provider the provider performing the validation
+ * @param instanceDefinition the instance definition
+ * @throws SliderException if invalid.
+ */
+ protected void validateInstanceDefinition(AbstractClientProvider provider,
+ AggregateConf instanceDefinition) throws SliderException {
+try {
+ provider.validateInstanceDefinition(instanceDefinition);
+} catch (SliderException e) {
+ //problem, reject it
+ log.info("Error {} validating application instance definition ", e);
+ log.debug("Error validating application instance definition ", e);
+ log.info(instanceDefinition.toString());
+ throw e;
+}
+ }
+
/**
* Load the persistent cluster description
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1deafee8/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 4b1b44f..bba3ee2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.
[3/7] git commit: SLIDER-476 make AM queue executor single thread
SLIDER-476 make AM queue executor single thread
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ddde0300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ddde0300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ddde0300
Branch: refs/heads/develop
Commit: ddde0300d3f17fbaa4e0dbdd4d820b18dffc99f9
Parents: 1deafee
Author: Steve Loughran
Authored: Thu Oct 2 13:51:52 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 13:51:52 2014 -0700
--
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ddde0300/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 365eeab..24d5eea 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -197,7 +197,6 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
- public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/**
* Singleton of metrics registry
@@ -212,7 +211,6 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
private AMRMClientAsync asyncRMClient;
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-
private RMOperationHandler rmOperationHandler;
private RMOperationHandler providerRMOperationHandler;
@@ -411,11 +409,10 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newCachedThreadPool(
+Executors.newSingleThreadExecutor(
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
-
addService(actionQueues);
//init all child services
super.serviceInit(conf);
[6/7] git commit: SLIDER-470 slider appears to support negative component counts: flex checks and in AM (not tested that)
SLIDER-470 slider appears to support negative component counts: flex checks and
in AM (not tested that)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cbf3a634
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cbf3a634
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cbf3a634
Branch: refs/heads/develop
Commit: cbf3a634b82be14c714d05d59b10ee9959fe08f6
Parents: 483ab95
Author: Steve Loughran
Authored: Thu Oct 2 14:59:02 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:59:02 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 18 ++--
.../apache/slider/core/conf/AggregateConf.java | 20 +
.../apache/slider/providers/ProviderCore.java | 4 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 94
.../TestFreezeThawMasterlessAM.groovy | 92 ---
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
...ndaloneClusterNegativeComponentCounts.groovy | 79
.../slider/providers/agent/TestAgentEcho.groovy | 18 +++-
8 files changed, 140 insertions(+), 187 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 05ef532..dc81f98 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2053,29 +2053,24 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
clusterDirectory);
-SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
-// provider to validate what there is
-SliderAMClientProvider provider = sliderAM;
-validateInstanceDefinition(provider, instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
for (Map.Entry entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- if (count < 0) {
-throw new BadCommandArgumentsException("Requested number of " + role
-+ " instances is out of range");
- }
resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
Integer.toString(count));
-
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
role,
count,
resources);
}
+SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+// slider provider to validate what there is
+validateInstanceDefinition(sliderAM, instanceDefinition);
+
int exitCode = EXIT_FALSE;
// save the specification
try {
@@ -2083,10 +2078,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
- log.warn("Failed to save new resource definition to {} : {}",
clusterDirectory,
- e.toString());
-
-
+ log.warn("Failed to save new resource definition to {} : {}",
clusterDirectory, e);
}
// now see if it is actually running and tell it about the update if it is
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 02669cd..e9e0dc9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -23,6 +23,8 @@ import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import java.io.IOException;
+
/**
* Aggregate Configuration.
*
@@ -58,6 +60,24 @@ public final class AggregateConf {
setInternal(internal);
}
+ /**
+ * Take a snapshot of the configuration
+ * @param instanceDefinition source
+ * @throws IOException marshalling/copying problems
+ */
+ public AggregateConf(AggregateConf instanceDefinition) throws IOException {
+
[4/7] git commit: SLIDER-476 make AM queue executor non-reentrant
SLIDER-476 make AM queue executor non-reentrant
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/755bb7ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/755bb7ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/755bb7ce
Branch: refs/heads/develop
Commit: 755bb7ce951a97fbbbca420429b6a254576d2b38
Parents: ddde030
Author: Steve Loughran
Authored: Thu Oct 2 14:24:46 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:24:46 2014 -0700
--
.../server/appmaster/SliderAppMaster.java | 49 +---
.../appmaster/actions/ActionStopQueue.java | 8 +++-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../server/appmaster/actions/QueueExecutor.java | 2 +
4 files changed, 42 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/755bb7ce/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 24d5eea..cc4f6fe 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -409,7 +409,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newSingleThreadExecutor(
+Executors.newCachedThreadPool(
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
@@ -1242,18 +1242,24 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
* It should be the only way that anything -even the AM itself on startup-
* asks for nodes.
* @param resources the resource tree
- * @throws IOException
+ * @throws SliderException slider problems, including invalid configs
+ * @throws IOException IO problems
*/
private void flexCluster(ConfTree resources)
-throws IOException, SliderInternalStateException, BadConfigException {
+ throws IOException, SliderException {
+
+AggregateConf newConf =
+new AggregateConf(appState.getInstanceDefinitionSnapshot());
+newConf.setResources(resources);
+// verify the new definition is valid
+sliderAMProvider.validateInstanceDefinition(newConf);
+providerService.validateInstanceDefinition(newConf);
appState.updateResourceDefinitions(resources);
// reset the scheduled windows...the values
// may have changed
appState.resetFailureCounts();
-
-
// ask for more containers if needed
reviewRequestAndReleaseNodes("flexCluster");
@@ -1416,11 +1422,23 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
/* SliderClusterProtocol */
/* === */
+ /**
+ * General actions to perform on a slider RPC call coming in
+ * @param operation operation to log
+ * @throws IOException problems
+ */
+ protected void onRpcCall(String operation) throws IOException {
+// it's not clear why this is here —it has been present since the
+// code -> git change. Leaving it in
+SliderUtils.getCurrentUser();
+log.debug("Received call to {}", operation);
+ }
+
@Override //SliderClusterProtocol
public Messages.StopClusterResponseProto
stopCluster(Messages.StopClusterRequestProto request) throws
IOException,
YarnException {
-SliderUtils.getCurrentUser();
+onRpcCall("stopCluster()");
String message = request.getMessage();
log.info("SliderAppMasterApi.stopCluster: {}", message);
schedule(new ActionStopSlider(message, 1000, TimeUnit.MILLISECONDS));
@@ -1431,8 +1449,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
public Messages.FlexClusterResponseProto
flexCluster(Messages.FlexClusterRequestProto request) throws
IOException,
YarnException {
-SliderUtils.getCurrentUser();
-
+onRpcCall("flexCluster()");
String payload = request.getClusterSpec();
ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
ConfTre
git commit: SLIDER-484 app-package/hbase not in slider base pom
Repository: incubator-slider
Updated Branches:
refs/heads/develop fef98cd64 -> e631a1323
SLIDER-484 app-package/hbase not in slider base pom
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e631a132
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e631a132
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e631a132
Branch: refs/heads/develop
Commit: e631a13233af331803f50f64fd30272affe6f47c
Parents: fef98cd
Author: Steve Loughran
Authored: Fri Oct 3 15:21:00 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:21:00 2014 -0700
--
app-packages/accumulo/pom.xml | 2 +-
app-packages/hbase/pom.xml| 2 +-
pom.xml | 2 ++
3 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index bc4b591..f2f1a35 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -74,7 +74,7 @@
org.apache.maven.plugins
maven-antrun-plugin
-1.7
+${maven-antrun-plugin.version}
copy
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/app-packages/hbase/pom.xml
--
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 519d7c1..be1b858 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -45,7 +45,7 @@
org.apache.maven.plugins
maven-antrun-plugin
-1.7
+${maven-antrun-plugin.version}
copy
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/pom.xml
--
diff --git a/pom.xml b/pom.xml
index b31d476..6d35407 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,6 +39,7 @@
slider-core
slider-agent
app-packages/accumulo
+app-packages/hbase
slider-assembly
slider-funtest
slider-providers/hbase/slider-hbase-provider
@@ -193,6 +194,7 @@
[3.0.0,)
+1.7
2.4
2.5.2
3.1
git commit: SLIDER-482 accumulo funtest to include registry retrieval
Repository: incubator-slider
Updated Branches:
refs/heads/develop e631a1323 -> fa0acf1f6
SLIDER-482 accumulo funtest to include registry retrieval
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa0acf1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa0acf1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa0acf1f
Branch: refs/heads/develop
Commit: fa0acf1f69a43119fd650b16b80c51cb0974975c
Parents: e631a13
Author: Steve Loughran
Authored: Fri Oct 3 15:22:33 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:22:33 2014 -0700
--
.../org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy| 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa0acf1f/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 12c0655..5ebce0a 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -167,6 +167,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
public static String getMonitorUrl(SliderClient sliderClient, String
clusterName) {
int tries = 5
+Exception caught;
while (true) {
try {
CuratorServiceInstance instance =
@@ -181,6 +182,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
assertNotNull monitorUrl
return monitorUrl
} catch (Exception e) {
+caught = e;
log.info("Got exception trying to read quicklinks")
if (tries-- == 0) {
break
@@ -188,7 +190,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
sleep(2)
}
}
-fail("Couldn't retrieve quicklinks")
+throw caught;
}
public static void checkMonitorPage(String monitorUrl) {
git commit: SLIDER-486 slider bin internally uses --classpath and causes issues in some JDKs
Repository: incubator-slider
Updated Branches:
refs/heads/develop 757f69fbf -> ed189c3e5
SLIDER-486 slider bin internally uses --classpath and causes issues in some JDKs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ed189c3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ed189c3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ed189c3e
Branch: refs/heads/develop
Commit: ed189c3e55ad6aedd665dce2c51803d556386505
Parents: 757f69f
Author: Steve Loughran
Authored: Sat Oct 4 14:23:06 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 14:23:06 2014 -0700
--
slider-assembly/src/main/scripts/slider | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed189c3e/slider-assembly/src/main/scripts/slider
--
diff --git a/slider-assembly/src/main/scripts/slider
b/slider-assembly/src/main/scripts/slider
index 7e277c5..c24bb92 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -73,7 +73,7 @@ echo "classpath = \"${slider_classpath}\""
export CLASSPATH="${slider_classpath}"
echo ""
-echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\"
${launcher} $@"
+echo "command is java ${slider_jvm_opts} -classpath \"${slider_classpath}\"
${launcher} $@"
echo ""
echo ""
type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin/
[2/5] git commit: Merge branch 'develop' into feature/SLIDER-467_FinalStatus_should_not_say_Failed_for_applications_that_are_stopped
Merge branch 'develop' into feature/SLIDER-467_FinalStatus_should_not_say_Failed_for_applications_that_are_stopped Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a53f559 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a53f559 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a53f559 Branch: refs/heads/develop Commit: 4a53f559cb1c8ca5ea594bc15d2d709226661a1c Parents: 1a3e45b fa0acf1 Author: Steve Loughran Authored: Fri Oct 3 16:24:29 2014 -0700 Committer: Steve Loughran Committed: Fri Oct 3 16:24:29 2014 -0700 -- app-packages/accumulo/pom.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 4 +- .../slider/funtest/accumulo/CertUtil.java | 50 ++-- app-packages/hbase/pom.xml | 2 +- pom.xml | 2 + 5 files changed, 52 insertions(+), 8 deletions(-) --
[4/5] git commit: SLIDER-467 final status to be "succeeded"...reworking of how exit codes are propagated with tests fixed where the changes showed they were incomplete/broken
SLIDER-467 final status to be "succeeded"...reworking of how exit codes are
propagated with tests fixed where the changes showed they were incomplete/broken
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3944b1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3944b1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3944b1c
Branch: refs/heads/develop
Commit: a3944b1cd44024acd4578a7e49f8abf0e8497f92
Parents: 155262b
Author: Steve Loughran
Authored: Sat Oct 4 19:48:56 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 19:48:56 2014 -0700
--
.../slider/core/exceptions/SliderException.java | 6 +--
.../apache/slider/providers/ProviderUtils.java | 6 ++-
.../server/appmaster/SliderAppMaster.java | 40 ++--
.../agent/actions/TestActionStatus.groovy | 9 ++---
.../standalone/TestStandaloneAMRestart.groovy | 38 ++-
5 files changed, 67 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
index 18e3157..7f3134a 100644
---
a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++
b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -41,9 +41,9 @@ public class SliderException extends ServiceLaunchException
implements
/**
* Format the exception as you create it
- * @param code
- * @param message
- * @param args
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
*/
public SliderException(int code, String message, Object... args) {
super(code, String.format(message, args));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 464aba5..8e77a9c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -204,11 +204,13 @@ public class ProviderUtils implements RoleKeys {
/**
* Validate the node count and heap size values of a node class
- *
+ *
+ * If max <= 0: min <= count
+ * If max > 0: min <= count <= max
* @param name node class name
* @param count requested node count
* @param min requested heap size
- * @param max
+ * @param max maximum value.
* @throws BadCommandArgumentsException if the values are out of range
*/
public void validateNodeCount(String name,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7fbea86..5676f3f 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -51,6 +51,7 @@ import
org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -86,6 +87,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
@@ -335,7
[1/5] git commit: SLIDER-467 final status == SUCCEEDED on shut down app
Repository: incubator-slider
Updated Branches:
refs/heads/develop 1893c7cbb -> 70b4b7592
SLIDER-467 final status == SUCCEEDED on shut down app
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1a3e45bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1a3e45bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1a3e45bc
Branch: refs/heads/develop
Commit: 1a3e45bce82dd36a98e0228648e358f3ea6e0b49
Parents: 2051dd6
Author: Steve Loughran
Authored: Fri Oct 3 15:08:52 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:08:52 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 24 ++-
.../TriggerClusterTeardownException.java| 16 +-
.../apache/slider/server/appmaster/AMUtils.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 191 +++
.../appmaster/actions/ActionStopSlider.java | 98 +-
.../server/appmaster/actions/QueueExecutor.java | 8 +
.../server/appmaster/actions/QueueService.java | 22 ++-
.../slider/server/appmaster/state/AppState.java | 8 +-
.../agent/actions/TestActionStatus.groovy | 7 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 13 +-
.../standalone/TestStandaloneAMDestroy.groovy | 17 +-
.../standalone/TestStandaloneAMRestart.groovy | 29 ++-
.../TestMockAppStateContainerFailure.groovy | 1 -
.../slider/test/YarnMiniClusterTestBase.groovy | 24 ++-
.../accumulo/live/TestAccFreezeThaw.groovy | 7 +-
15 files changed, 337 insertions(+), 131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a3e45bc/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dc81f98..d1a1a7b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1545,7 +1545,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
/**
- * Kill the submitted application by sending a call to the ASM
+ * Kill the submitted application via YARN
* @throws YarnException
* @throws IOException
*/
@@ -1875,7 +1875,6 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
LaunchedApplication application = new LaunchedApplication(yarnClient, app);
applicationId = application.getApplicationId();
-
if (forcekill) {
//escalating to forced kill
application.kill("Forced stop of " + clustername +
@@ -2423,7 +2422,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @throws IOException
* Network or other problems
*/
- private int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
try {
if (diagnosticArgs.client) {
actionDiagnosticClient();
@@ -2465,7 +2464,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// we are catching exceptions here because those are
indication of
// validation result, and we need to print them here
log.error("validation of slider-client.xml fails
because: "
- + e.toString());
+ + e.toString(), e);
return;
}
SliderClusterOperations clusterOperations =
createClusterOperations(clusterName);
@@ -2489,19 +2488,24 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
try {
SliderUtils.validateHDFSFile(sliderFileSystem,
imagePath);
- log.info("Slider agent tarball is properly
installed");
+ log.info("Slider agent package is properly
installed");
+ } catch (FileNotFoundException e) {
+ log.error("can not find agent package: {}", e);
+ return;
} catch (IOException e) {
- log.error("can not find or open agent tar ball:
" + e.toString());
+ log.error("can not open agent package: {}", e,
e);
return;
}
[3/5] git commit: SLIDER-467 cleanly shut down AMs should say "succeeded" in app reports. Easier said than done
SLIDER-467 cleanly shut down AMs should say "succeeded" in app reports. Easier
said than done
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/155262bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/155262bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/155262bf
Branch: refs/heads/develop
Commit: 155262bf7555139cc54d553e6ef34e612d075a6f
Parents: 4a53f55
Author: Steve Loughran
Authored: Sat Oct 4 14:02:48 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 14:02:48 2014 -0700
--
.../providers/AbstractProviderService.java | 21 ++-
.../providers/agent/AgentProviderService.java | 22
.../slideram/SliderAMProviderService.java | 8 -
.../apache/slider/server/appmaster/AMUtils.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 38 +---
.../appmaster/actions/ActionStopSlider.java | 20 +++
.../workflow/WorkflowSequenceService.java | 8 -
.../standalone/TestStandaloneAMDestroy.groovy | 3 +-
.../standalone/TestStandaloneAMRestart.groovy | 36 +--
.../slider/test/YarnMiniClusterTestBase.groovy | 13 ---
.../providers/hbase/HBaseProviderService.java | 23
11 files changed, 99 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index baddb56..b3cf527 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -77,6 +77,7 @@ public abstract class AbstractProviderService
public AbstractProviderService(String name) {
super(name);
+setStopIfNoChildServicesAtStartup(false);
}
@Override
@@ -184,7 +185,25 @@ public abstract class AbstractProviderService
}
return false;
}
-
+
+ /**
+ * override point to allow a process to start executing in this container
+ * @param instanceDefinition cluster description
+ * @param confDir configuration directory
+ * @param env environment
+ * @param execInProgress the callback for the exec events
+ * @return false
+ * @throws IOException
+ * @throws SliderException
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map env,
+ ProviderCompleted execInProgress) throws IOException, SliderException {
+return false;
+ }
+
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@Override // ExitCodeProvider
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d7943b2..67a268e 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -395,28 +395,6 @@ public class AgentProviderService extends
AbstractProviderService implements
}
}
- /**
- * Run this service
- *
- * @param instanceDefinition component description
- * @param confDirlocal dir with the config
- * @param envenvironment variables above those generated by
- * @param execInProgress callback for the event notification
- *
- * @throws IOException IO problems
- * @throws SliderException anything internal
- */
- @Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map env,
- ProviderCompleted execInProgress) throws
- IOException,
- SliderException {
-
-return false;
- }
-
@Override
public boolean isSupportedRole(String role) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
b/slider-cor
[5/5] git commit: Merge branch 'feature/SLIDER-467_FinalStatus' into develop
Merge branch 'feature/SLIDER-467_FinalStatus' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/70b4b759 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/70b4b759 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/70b4b759 Branch: refs/heads/develop Commit: 70b4b75927bdf56184bb6d6a8e2bb5fafbf4c50c Parents: 1893c7c a3944b1 Author: Steve Loughran Authored: Sun Oct 5 17:31:53 2014 -0700 Committer: Steve Loughran Committed: Sun Oct 5 17:31:53 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 24 +- .../slider/core/exceptions/SliderException.java | 6 +- .../TriggerClusterTeardownException.java| 16 +- .../providers/AbstractProviderService.java | 21 +- .../apache/slider/providers/ProviderUtils.java | 6 +- .../providers/agent/AgentProviderService.java | 22 -- .../slideram/SliderAMProviderService.java | 8 - .../apache/slider/server/appmaster/AMUtils.java | 5 +- .../server/appmaster/SliderAppMaster.java | 249 +++ .../appmaster/actions/ActionStopSlider.java | 116 - .../server/appmaster/actions/QueueExecutor.java | 8 + .../server/appmaster/actions/QueueService.java | 22 +- .../slider/server/appmaster/state/AppState.java | 8 +- .../workflow/WorkflowSequenceService.java | 8 +- .../agent/actions/TestActionStatus.groovy | 16 +- .../TestFreezeThawFlexStandaloneAM.groovy | 13 +- .../standalone/TestStandaloneAMDestroy.groovy | 20 +- .../standalone/TestStandaloneAMRestart.groovy | 77 -- .../TestMockAppStateContainerFailure.groovy | 1 - .../slider/test/YarnMiniClusterTestBase.groovy | 37 ++- .../accumulo/live/TestAccFreezeThaw.groovy | 7 +- .../providers/hbase/HBaseProviderService.java | 23 -- 22 files changed, 479 insertions(+), 234 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/70b4b759/slider-core/src/main/java/org/apache/slider/client/SliderClient.java --
[19/24] git commit: SLIDER-467 cleanly shut down AMs should say "succeeded" in app reports. Easier said than done
SLIDER-467 cleanly shut down AMs should say "succeeded" in app reports. Easier
said than done
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/155262bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/155262bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/155262bf
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 155262bf7555139cc54d553e6ef34e612d075a6f
Parents: 4a53f55
Author: Steve Loughran
Authored: Sat Oct 4 14:02:48 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 14:02:48 2014 -0700
--
.../providers/AbstractProviderService.java | 21 ++-
.../providers/agent/AgentProviderService.java | 22
.../slideram/SliderAMProviderService.java | 8 -
.../apache/slider/server/appmaster/AMUtils.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 38 +---
.../appmaster/actions/ActionStopSlider.java | 20 +++
.../workflow/WorkflowSequenceService.java | 8 -
.../standalone/TestStandaloneAMDestroy.groovy | 3 +-
.../standalone/TestStandaloneAMRestart.groovy | 36 +--
.../slider/test/YarnMiniClusterTestBase.groovy | 13 ---
.../providers/hbase/HBaseProviderService.java | 23
11 files changed, 99 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index baddb56..b3cf527 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -77,6 +77,7 @@ public abstract class AbstractProviderService
public AbstractProviderService(String name) {
super(name);
+setStopIfNoChildServicesAtStartup(false);
}
@Override
@@ -184,7 +185,25 @@ public abstract class AbstractProviderService
}
return false;
}
-
+
+ /**
+ * override point to allow a process to start executing in this container
+ * @param instanceDefinition cluster description
+ * @param confDir configuration directory
+ * @param env environment
+ * @param execInProgress the callback for the exec events
+ * @return false
+ * @throws IOException
+ * @throws SliderException
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map env,
+ ProviderCompleted execInProgress) throws IOException, SliderException {
+return false;
+ }
+
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@Override // ExitCodeProvider
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d7943b2..67a268e 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -395,28 +395,6 @@ public class AgentProviderService extends
AbstractProviderService implements
}
}
- /**
- * Run this service
- *
- * @param instanceDefinition component description
- * @param confDirlocal dir with the config
- * @param envenvironment variables above those generated by
- * @param execInProgress callback for the event notification
- *
- * @throws IOException IO problems
- * @throws SliderException anything internal
- */
- @Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map env,
- ProviderCompleted execInProgress) throws
- IOException,
- SliderException {
-
-return false;
- }
-
@Override
public boolean isSupportedRole(String role) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/155262bf/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/slideram/
[20/24] git commit: SLIDER-486 slider bin internally uses --classpath and causes issues in some JDKs
SLIDER-486 slider bin internally uses --classpath and causes issues in some JDKs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ed189c3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ed189c3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ed189c3e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ed189c3e55ad6aedd665dce2c51803d556386505
Parents: 757f69f
Author: Steve Loughran
Authored: Sat Oct 4 14:23:06 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 14:23:06 2014 -0700
--
slider-assembly/src/main/scripts/slider | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed189c3e/slider-assembly/src/main/scripts/slider
--
diff --git a/slider-assembly/src/main/scripts/slider
b/slider-assembly/src/main/scripts/slider
index 7e277c5..c24bb92 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -73,7 +73,7 @@ echo "classpath = \"${slider_classpath}\""
export CLASSPATH="${slider_classpath}"
echo ""
-echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\"
${launcher} $@"
+echo "command is java ${slider_jvm_opts} -classpath \"${slider_classpath}\"
${launcher} $@"
echo ""
echo ""
type java >/dev/null 2>&1 || javabin=${JAVA_HOME}/bin/
[15/24] git commit: Merge branch 'develop' into feature/SLIDER-467_FinalStatus_should_not_say_Failed_for_applications_that_are_stopped
Merge branch 'develop' into feature/SLIDER-467_FinalStatus_should_not_say_Failed_for_applications_that_are_stopped Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a53f559 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a53f559 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a53f559 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 4a53f559cb1c8ca5ea594bc15d2d709226661a1c Parents: 1a3e45b fa0acf1 Author: Steve Loughran Authored: Fri Oct 3 16:24:29 2014 -0700 Committer: Steve Loughran Committed: Fri Oct 3 16:24:29 2014 -0700 -- app-packages/accumulo/pom.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 4 +- .../slider/funtest/accumulo/CertUtil.java | 50 ++-- app-packages/hbase/pom.xml | 2 +- pom.xml | 2 + 5 files changed, 52 insertions(+), 8 deletions(-) --
[07/24] git commit: SLIDER-476 make AM queue executor non-reentrant
SLIDER-476 make AM queue executor non-reentrant
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/755bb7ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/755bb7ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/755bb7ce
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 755bb7ce951a97fbbbca420429b6a254576d2b38
Parents: ddde030
Author: Steve Loughran
Authored: Thu Oct 2 14:24:46 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:24:46 2014 -0700
--
.../server/appmaster/SliderAppMaster.java | 49 +---
.../appmaster/actions/ActionStopQueue.java | 8 +++-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../server/appmaster/actions/QueueExecutor.java | 2 +
4 files changed, 42 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/755bb7ce/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 24d5eea..cc4f6fe 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -409,7 +409,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newSingleThreadExecutor(
+Executors.newCachedThreadPool(
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
@@ -1242,18 +1242,24 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
* It should be the only way that anything -even the AM itself on startup-
* asks for nodes.
* @param resources the resource tree
- * @throws IOException
+ * @throws SliderException slider problems, including invalid configs
+ * @throws IOException IO problems
*/
private void flexCluster(ConfTree resources)
-throws IOException, SliderInternalStateException, BadConfigException {
+ throws IOException, SliderException {
+
+AggregateConf newConf =
+new AggregateConf(appState.getInstanceDefinitionSnapshot());
+newConf.setResources(resources);
+// verify the new definition is valid
+sliderAMProvider.validateInstanceDefinition(newConf);
+providerService.validateInstanceDefinition(newConf);
appState.updateResourceDefinitions(resources);
// reset the scheduled windows...the values
// may have changed
appState.resetFailureCounts();
-
-
// ask for more containers if needed
reviewRequestAndReleaseNodes("flexCluster");
@@ -1416,11 +1422,23 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
/* SliderClusterProtocol */
/* === */
+ /**
+ * General actions to perform on a slider RPC call coming in
+ * @param operation operation to log
+ * @throws IOException problems
+ */
+ protected void onRpcCall(String operation) throws IOException {
+// it's not clear why this is here —it has been present since the
+// code -> git change. Leaving it in
+SliderUtils.getCurrentUser();
+log.debug("Received call to {}", operation);
+ }
+
@Override //SliderClusterProtocol
public Messages.StopClusterResponseProto
stopCluster(Messages.StopClusterRequestProto request) throws
IOException,
YarnException {
-SliderUtils.getCurrentUser();
+onRpcCall("stopCluster()");
String message = request.getMessage();
log.info("SliderAppMasterApi.stopCluster: {}", message);
schedule(new ActionStopSlider(message, 1000, TimeUnit.MILLISECONDS));
@@ -1431,8 +1449,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
public Messages.FlexClusterResponseProto
flexCluster(Messages.FlexClusterRequestProto request) throws
IOException,
YarnException {
-SliderUtils.getCurrentUser();
-
+onRpcCall("flexCluster()");
String payload = request.getClusterSpec();
ConfTreeSerDeser confTreeSerD
[21/24] git commit: SLIDER-467 final status to be "succeeded"...reworking of how exit codes are propagated with tests fixed where the changes showed they were incomplete/broken
SLIDER-467 final status to be "succeeded"...reworking of how exit codes are
propagated with tests fixed where the changes showed they were incomplete/broken
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3944b1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3944b1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3944b1c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a3944b1cd44024acd4578a7e49f8abf0e8497f92
Parents: 155262b
Author: Steve Loughran
Authored: Sat Oct 4 19:48:56 2014 -0700
Committer: Steve Loughran
Committed: Sat Oct 4 19:48:56 2014 -0700
--
.../slider/core/exceptions/SliderException.java | 6 +--
.../apache/slider/providers/ProviderUtils.java | 6 ++-
.../server/appmaster/SliderAppMaster.java | 40 ++--
.../agent/actions/TestActionStatus.groovy | 9 ++---
.../standalone/TestStandaloneAMRestart.groovy | 38 ++-
5 files changed, 67 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
index 18e3157..7f3134a 100644
---
a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++
b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -41,9 +41,9 @@ public class SliderException extends ServiceLaunchException
implements
/**
* Format the exception as you create it
- * @param code
- * @param message
- * @param args
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
*/
public SliderException(int code, String message, Object... args) {
super(code, String.format(message, args));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 464aba5..8e77a9c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -204,11 +204,13 @@ public class ProviderUtils implements RoleKeys {
/**
* Validate the node count and heap size values of a node class
- *
+ *
+ * If max <= 0: min <= count
+ * If max > 0: min <= count <= max
* @param name node class name
* @param count requested node count
* @param min requested heap size
- * @param max
+ * @param max maximum value.
* @throws BadCommandArgumentsException if the values are out of range
*/
public void validateNodeCount(String name,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3944b1c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7fbea86..5676f3f 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -51,6 +51,7 @@ import
org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -86,6 +87,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slide
[11/24] git commit: SLIDER-374 Potential resource leak in accumulo/CertUtil.java due to unclosed streams (Thomas Liu)
SLIDER-374 Potential resource leak in accumulo/CertUtil.java due to unclosed
streams (Thomas Liu)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fef98cd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fef98cd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fef98cd6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: fef98cd64a73b7e4e215e810f18c99c4b942afee
Parents: 2051dd6
Author: tedyu
Authored: Fri Oct 3 11:22:30 2014 -0700
Committer: tedyu
Committed: Fri Oct 3 11:22:30 2014 -0700
--
.../slider/funtest/accumulo/CertUtil.java | 50 ++--
1 file changed, 45 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fef98cd6/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
--
diff --git
a/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
b/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
index 4aba31f..8bac58f 100644
---
a/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
+++
b/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
@@ -97,13 +97,29 @@ public class CertUtil {
IOException, CertificateException, NoSuchAlgorithmException {
KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
char[] signerPasswordArray = rootKeystorePassword;
-signerKeystore.load(new FileInputStream(rootKeystorePath),
signerPasswordArray);
+FileInputStream rootKeystoreInputStream = null;
+try{
+rootKeystoreInputStream = new FileInputStream(rootKeystorePath);
+signerKeystore.load(rootKeystoreInputStream, signerPasswordArray);
+} finally {
+if(rootKeystoreInputStream != null) {
+rootKeystoreInputStream.close();
+}
+}
Certificate rootCert = findCert(signerKeystore);
KeyStore keystore = KeyStore.getInstance(keystoreType);
keystore.load(null, null);
keystore.setCertificateEntry(keyName + "Cert", rootCert);
-keystore.store(new FileOutputStream(targetKeystoreFile),
truststorePassword);
+FileOutputStream targetKeystoreOutputStream = null;
+try{
+targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+keystore.store(targetKeystoreOutputStream, truststorePassword);
+} finally {
+if(targetKeystoreOutputStream != null) {
+targetKeystoreOutputStream.close();
+}
+}
}
public static void createSignedCert(String targetKeystoreFile,
@@ -112,7 +128,15 @@ public class CertUtil {
throws Exception {
KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
char[] signerPasswordArray = signerKeystorePassword;
-signerKeystore.load(new FileInputStream(signerKeystorePath),
signerPasswordArray);
+FileInputStream signerKeystoreInputStream = null;
+try{
+signerKeystoreInputStream = new FileInputStream(signerKeystorePath);
+signerKeystore.load(signerKeystoreInputStream, signerPasswordArray);
+} finally {
+if (signerKeystoreInputStream != null) {
+signerKeystoreInputStream.close();
+}
+}
Certificate signerCert = findCert(signerKeystore);
PrivateKey signerKey = findPrivateKey(signerKeystore, signerPasswordArray);
@@ -125,7 +149,15 @@ public class CertUtil {
keystore.load(null, null);
keystore.setCertificateEntry(keyName + "Cert", cert);
keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new
Certificate[] {cert, signerCert});
-keystore.store(new FileOutputStream(targetKeystoreFile), password);
+FileOutputStream targetKeystoreOutputStream = null;
+try{
+targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+keystore.store(targetKeystoreOutputStream, password);
+} finally {
+if (targetKeystoreOutputStream != null){
+targetKeystoreOutputStream.close();
+}
+}
}
public static void createSelfSignedCert(String targetKeystoreFileName,
@@ -148,7 +180,15 @@ public class CertUtil {
keystore.load(null, null);
keystore.setCertificateEntry(keyName + "Cert", cert);
keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new
Certificate[] {cert});
-keystore.store(new FileOutputStream(targetKeystoreFile), password);
+FileOutputStream targetKeystoreOutputStream = null;
+try{
+targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+keystore.store(targetKeystoreOutputStream, password);
+} finally {
+
[08/24] git commit: SLIDER-476 make AM queue executor non-reentrant
SLIDER-476 make AM queue executor non-reentrant
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/483ab958
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/483ab958
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/483ab958
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 483ab958956f52177205ea928730e4bf71935b34
Parents: 755bb7c
Author: Steve Loughran
Authored: Thu Oct 2 14:25:36 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:25:36 2014 -0700
--
.../java/org/apache/slider/server/appmaster/SliderAppMaster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/483ab958/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index cc4f6fe..74a6f34 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -409,7 +409,7 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newCachedThreadPool(
+Executors.newFixedThreadPool(2,
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
[10/24] git commit: Merge branch 'feature/SLIDER-470_slider_appears_to_support_negative_component_counts' into develop
Merge branch 'feature/SLIDER-470_slider_appears_to_support_negative_component_counts' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2051dd64 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2051dd64 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2051dd64 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 2051dd6411b47c2ec3ea6de0cf3b77ddc054df22 Parents: 6b011d9 cbf3a63 Author: Steve Loughran Authored: Thu Oct 2 14:59:14 2014 -0700 Committer: Steve Loughran Committed: Thu Oct 2 14:59:14 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 48 +- .../apache/slider/core/conf/AggregateConf.java | 20 + .../apache/slider/core/conf/MapOperations.java | 16 +++- .../providers/AbstractClientProvider.java | 9 +- .../apache/slider/providers/ProviderCore.java | 9 +- .../slideram/SliderAMClientProvider.java| 28 ++ .../server/appmaster/SliderAppMaster.java | 52 +++ .../appmaster/actions/ActionStopQueue.java | 8 +- .../server/appmaster/actions/AsyncAction.java | 2 +- .../server/appmaster/actions/QueueExecutor.java | 2 + .../slider/server/appmaster/state/AppState.java | 5 +- .../server/appmaster/state/RoleStatus.java | 20 + .../TestFreezeThawFlexStandaloneAM.groovy | 94 .../TestFreezeThawMasterlessAM.groovy | 92 --- .../standalone/TestBuildStandaloneAM.groovy | 25 -- .../standalone/TestStandaloneAgentAM.groovy | 2 +- .../slider/providers/agent/TestAgentEcho.groovy | 18 +++- .../build/TestBuildThawClusterM1W1.groovy | 3 +- 18 files changed, 298 insertions(+), 155 deletions(-) --
[14/24] git commit: SLIDER-482 accumulo funtest to include registry retrieval
SLIDER-482 accumulo funtest to include registry retrieval
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa0acf1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa0acf1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa0acf1f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: fa0acf1f69a43119fd650b16b80c51cb0974975c
Parents: e631a13
Author: Steve Loughran
Authored: Fri Oct 3 15:22:33 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:22:33 2014 -0700
--
.../org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy| 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa0acf1f/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 12c0655..5ebce0a 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -167,6 +167,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
public static String getMonitorUrl(SliderClient sliderClient, String
clusterName) {
int tries = 5
+Exception caught;
while (true) {
try {
CuratorServiceInstance instance =
@@ -181,6 +182,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
assertNotNull monitorUrl
return monitorUrl
} catch (Exception e) {
+caught = e;
log.info("Got exception trying to read quicklinks")
if (tries-- == 0) {
break
@@ -188,7 +190,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
sleep(2)
}
}
-fail("Couldn't retrieve quicklinks")
+throw caught;
}
public static void checkMonitorPage(String monitorUrl) {
[24/24] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Conflicts: slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c51f2f0e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c51f2f0e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c51f2f0e Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: c51f2f0eac6a7938e1f0acc27a4f5df90bb88339 Parents: 733745e 70b4b75 Author: Steve Loughran Authored: Sun Oct 5 19:56:47 2014 -0700 Committer: Steve Loughran Committed: Sun Oct 5 19:56:47 2014 -0700 -- .../accumulo/configuration/accumulo-site.xml| 19 +- app-packages/accumulo/configuration/global.xml | 7 +- app-packages/accumulo/pom.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 4 +- .../slider/funtest/accumulo/CertUtil.java | 50 +++- .../app-pkg-template/appConfig-default.json | 2 +- .../command-logger/slider-pkg/appConfig.json| 2 +- app-packages/hbase/appConfig-default.json | 2 +- app-packages/hbase/pom.xml | 2 +- app-packages/memcached/appConfig-default.json | 2 +- app-packages/storm-win/appConfig-default.json | 2 - app-packages/storm-win/metainfo.xml | 12 +- .../storm-win/package/scripts/params.py | 2 - .../storm-win/package/scripts/service.py| 1 + .../storm-win/package/scripts/status_params.py | 2 - .../storm-win/package/templates/config.yaml.j2 | 9 - app-packages/storm/appConfig-default.json | 4 +- app-packages/storm/metainfo.xml | 23 +- app-packages/storm/package/scripts/params.py| 2 - .../storm/package/templates/config.yaml.j2 | 9 - app-packages/storm/resources-default.json | 4 - pom.xml | 2 + slider-assembly/src/main/scripts/slider | 2 +- .../org/apache/slider/client/SliderClient.java | 77 +++-- .../apache/slider/common/SliderXmlConfKeys.java | 4 +- .../apache/slider/core/conf/AggregateConf.java | 20 ++ .../apache/slider/core/conf/MapOperations.java | 16 +- .../slider/core/exceptions/SliderException.java | 6 +- .../TriggerClusterTeardownException.java| 16 +- .../providers/AbstractClientProvider.java | 9 +- .../providers/AbstractProviderService.java | 21 +- .../apache/slider/providers/ProviderCore.java | 9 +- .../apache/slider/providers/ProviderUtils.java | 6 +- .../providers/agent/AgentProviderService.java | 28 +- .../providers/agent/ComponentInstanceState.java | 2 +- .../slideram/SliderAMClientProvider.java| 28 ++ .../slideram/SliderAMProviderService.java | 8 - .../apache/slider/server/appmaster/AMUtils.java | 5 +- .../server/appmaster/SliderAppMaster.java | 298 +++ .../appmaster/actions/ActionStopQueue.java | 8 +- .../appmaster/actions/ActionStopSlider.java | 116 +++- .../server/appmaster/actions/AsyncAction.java | 2 +- .../server/appmaster/actions/QueueExecutor.java | 10 + .../server/appmaster/actions/QueueService.java | 22 +- .../slider/server/appmaster/state/AppState.java | 13 +- .../server/appmaster/state/RoleStatus.java | 20 ++ .../workflow/WorkflowSequenceService.java | 8 +- .../agent/actions/TestActionStatus.groovy | 16 +- .../TestFreezeThawFlexStandaloneAM.groovy | 99 ++ .../TestFreezeThawMasterlessAM.groovy | 92 -- .../standalone/TestBuildStandaloneAM.groovy | 22 +- .../standalone/TestStandaloneAMDestroy.groovy | 20 +- .../TestStandaloneAMMonkeyRestart.groovy| 12 +- .../standalone/TestStandaloneAMRestart.groovy | 77 - .../standalone/TestStandaloneAgentAM.groovy | 2 +- .../slider/providers/agent/TestAgentEcho.groovy | 18 +- .../TestMockAppStateContainerFailure.groovy | 1 - .../slider/test/YarnMiniClusterTestBase.groovy | 37 ++- .../accumulo/live/TestAccFreezeThaw.groovy | 7 +- .../providers/hbase/HBaseProviderService.java | 23 -- .../build/TestBuildThawClusterM1W1.groovy | 3 +- 61 files changed, 857 insertions(+), 490 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy -- diff --cc app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index 7ddaf9f,5ebce0a..fd8af9a --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/Accum
[23/24] git commit: Merge branch 'feature/SLIDER-467_FinalStatus' into develop
Merge branch 'feature/SLIDER-467_FinalStatus' into develop Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/70b4b759 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/70b4b759 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/70b4b759 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 70b4b75927bdf56184bb6d6a8e2bb5fafbf4c50c Parents: 1893c7c a3944b1 Author: Steve Loughran Authored: Sun Oct 5 17:31:53 2014 -0700 Committer: Steve Loughran Committed: Sun Oct 5 17:31:53 2014 -0700 -- .../org/apache/slider/client/SliderClient.java | 24 +- .../slider/core/exceptions/SliderException.java | 6 +- .../TriggerClusterTeardownException.java| 16 +- .../providers/AbstractProviderService.java | 21 +- .../apache/slider/providers/ProviderUtils.java | 6 +- .../providers/agent/AgentProviderService.java | 22 -- .../slideram/SliderAMProviderService.java | 8 - .../apache/slider/server/appmaster/AMUtils.java | 5 +- .../server/appmaster/SliderAppMaster.java | 249 +++ .../appmaster/actions/ActionStopSlider.java | 116 - .../server/appmaster/actions/QueueExecutor.java | 8 + .../server/appmaster/actions/QueueService.java | 22 +- .../slider/server/appmaster/state/AppState.java | 8 +- .../workflow/WorkflowSequenceService.java | 8 +- .../agent/actions/TestActionStatus.groovy | 16 +- .../TestFreezeThawFlexStandaloneAM.groovy | 13 +- .../standalone/TestStandaloneAMDestroy.groovy | 20 +- .../standalone/TestStandaloneAMRestart.groovy | 77 -- .../TestMockAppStateContainerFailure.groovy | 1 - .../slider/test/YarnMiniClusterTestBase.groovy | 37 ++- .../accumulo/live/TestAccFreezeThaw.groovy | 7 +- .../providers/hbase/HBaseProviderService.java | 23 -- 22 files changed, 479 insertions(+), 234 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/70b4b759/slider-core/src/main/java/org/apache/slider/client/SliderClient.java --
[09/24] git commit: SLIDER-470 slider appears to support negative component counts: flex checks and in AM (not tested that)
SLIDER-470 slider appears to support negative component counts: flex checks and
in AM (not tested that)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cbf3a634
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cbf3a634
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cbf3a634
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: cbf3a634b82be14c714d05d59b10ee9959fe08f6
Parents: 483ab95
Author: Steve Loughran
Authored: Thu Oct 2 14:59:02 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 14:59:02 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 18 ++--
.../apache/slider/core/conf/AggregateConf.java | 20 +
.../apache/slider/providers/ProviderCore.java | 4 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 94
.../TestFreezeThawMasterlessAM.groovy | 92 ---
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
...ndaloneClusterNegativeComponentCounts.groovy | 79
.../slider/providers/agent/TestAgentEcho.groovy | 18 +++-
8 files changed, 140 insertions(+), 187 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 05ef532..dc81f98 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2053,29 +2053,24 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
clusterDirectory);
-SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
-// provider to validate what there is
-SliderAMClientProvider provider = sliderAM;
-validateInstanceDefinition(provider, instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
for (Map.Entry entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- if (count < 0) {
-throw new BadCommandArgumentsException("Requested number of " + role
-+ " instances is out of range");
- }
resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
Integer.toString(count));
-
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
role,
count,
resources);
}
+SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+// slider provider to validate what there is
+validateInstanceDefinition(sliderAM, instanceDefinition);
+
int exitCode = EXIT_FALSE;
// save the specification
try {
@@ -2083,10 +2078,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
- log.warn("Failed to save new resource definition to {} : {}",
clusterDirectory,
- e.toString());
-
-
+ log.warn("Failed to save new resource definition to {} : {}",
clusterDirectory, e);
}
// now see if it is actually running and tell it about the update if it is
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 02669cd..e9e0dc9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -23,6 +23,8 @@ import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import java.io.IOException;
+
/**
* Aggregate Configuration.
*
@@ -58,6 +60,24 @@ public final class AggregateConf {
setInternal(internal);
}
+ /**
+ * Take a snapshot of the configuration
+ * @param instanceDefinition source
+ * @throws IOException marshalling/copying problems
+ */
+ public AggregateConf(AggregateConf ins
[04/24] git commit: SLIDER-470 slider appears to support negative component counts
SLIDER-470 slider appears to support negative component counts
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ca99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ca99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ca99
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ca992abfcd17f9e0fc4f0c677fdbdb0c
Parents: 6b011d9
Author: Steve Loughran
Authored: Thu Oct 2 09:32:52 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 09:32:52 2014 -0700
--
...ndaloneClusterNegativeComponentCounts.groovy | 79
1 file changed, 79 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ca99/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
new file mode 100644
index 000..8a23dc5
--- /dev/null
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestStandaloneClusterNegativeComponentCounts
+extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testStandaloneClusterNegativeComponentCounts() throws Throwable {
+describe "Run a Standalone AM with cluster counts set to 0 ... expect
failures"
+// patch the configuration for AM restart
+int threshold = 2;
+YarnConfiguration conf = getRestartableConfiguration(threshold)
+
+String clustername = createMiniCluster("", conf, 1, true)
+if (hdfsCluster) {
+ fail("Agent tests do not (currently) work with mini HDFS cluster")
+}
+ServiceLauncher launcher =
+createCluster(clustername,
+["negative":-4],
+[
+],
+true,
+false,
+agentDefOptions)
+SliderClient sliderClient = launcher.service
+addToTeardown(sliderClient);
+
+ApplicationReport report
+report = waitForClusterLive(sliderClient, 3)
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+def conf = new YarnConfiguration(configuration)
+conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+conf
+ }
+
+
+}
[01/24] git commit: SLIDER-465 remove props from accumulo default configuration files
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 733745eaf ->
c51f2f0ea
SLIDER-465 remove props from accumulo default configuration files
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2bbaafd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2bbaafd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2bbaafd5
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2bbaafd5d7fcd352731dd165ce320125f274f7b5
Parents: 41ec741
Author: Billie Rinaldi
Authored: Wed Oct 1 09:22:32 2014 -0700
Committer: Billie Rinaldi
Committed: Wed Oct 1 09:22:32 2014 -0700
--
.../accumulo/configuration/accumulo-site.xml | 19 +--
app-packages/accumulo/configuration/global.xml | 7 +--
2 files changed, 2 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2bbaafd5/app-packages/accumulo/configuration/accumulo-site.xml
--
diff --git a/app-packages/accumulo/configuration/accumulo-site.xml
b/app-packages/accumulo/configuration/accumulo-site.xml
index 269cc2b..3001c45 100644
--- a/app-packages/accumulo/configuration/accumulo-site.xml
+++ b/app-packages/accumulo/configuration/accumulo-site.xml
@@ -28,17 +28,6 @@
-instance.secret
-DEFAULT
-A secret unique to a given instance that all servers
- must know in order to communicate with one another.
- Change it before initialization. To
- change it later use ./bin/accumulo
org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new
[newpasswd],
- and then update this file.
-
-
-
-
tserver.memory.maps.max
80M
@@ -54,12 +43,6 @@
-trace.token.property.password
-
-secret
-
-
-
trace.user
root
@@ -71,7 +54,7 @@
tserver.walog.max.size
-100M
+40M
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2bbaafd5/app-packages/accumulo/configuration/global.xml
--
diff --git a/app-packages/accumulo/configuration/global.xml
b/app-packages/accumulo/configuration/global.xml
index 5d39dca..904a645 100644
--- a/app-packages/accumulo/configuration/global.xml
+++ b/app-packages/accumulo/configuration/global.xml
@@ -58,15 +58,10 @@
accumulo_hdfs_root_dir
-/apps/accumulo/data
+${DEFAULT_DATA_DIR}
Accumulo Relative Path to HDFS.
-accumulo_conf_dir
-/etc/accumulo
-Config Directory for Accumulo.
-
-
app_user
yarn
Accumulo User Name.
[16/24] git commit: SLIDER-366. Cleanup application packages SLIDER-485. Slider should not set the default queue to value default. It should rely on YARN to pick the default queue
SLIDER-366. Cleanup application packages
SLIDER-485. Slider should not set the default queue to value default. It should
rely on YARN to pick the default queue
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0143c210
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0143c210
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0143c210
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0143c210da401225ac257c26c10ce4c38529d889
Parents: fa0acf1
Author: Sumit Mohanty
Authored: Fri Oct 3 18:33:31 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Oct 3 18:33:35 2014 -0700
--
app-packages/app-pkg-template/appConfig-default.json| 2 +-
app-packages/command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/memcached/appConfig-default.json | 2 +-
app-packages/storm/appConfig-default.json | 2 +-
.../src/main/java/org/apache/slider/client/SliderClient.java| 5 -
.../main/java/org/apache/slider/common/SliderXmlConfKeys.java | 4 ++--
.../apache/slider/providers/agent/ComponentInstanceState.java | 2 +-
8 files changed, 12 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0143c210/app-packages/app-pkg-template/appConfig-default.json
--
diff --git a/app-packages/app-pkg-template/appConfig-default.json
b/app-packages/app-pkg-template/appConfig-default.json
index 3908030..457e013 100644
--- a/app-packages/app-pkg-template/appConfig-default.json
+++ b/app-packages/app-pkg-template/appConfig-default.json
@@ -4,7 +4,7 @@
},
"global": {
"application.def": "myapp-1.0.0.zip",
-"java_home": "/usr/jdk64/jdk1.7.0_45",
+"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0143c210/app-packages/command-logger/slider-pkg/appConfig.json
--
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json
b/app-packages/command-logger/slider-pkg/appConfig.json
index 73bc7fa..c8bb237 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -4,7 +4,7 @@
},
"global": {
"application.def":
".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
-"java_home": "/usr/jdk64/jdk1.7.0_45",
+"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0143c210/app-packages/hbase/appConfig-default.json
--
diff --git a/app-packages/hbase/appConfig-default.json
b/app-packages/hbase/appConfig-default.json
index c7952df..658d727 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -5,7 +5,7 @@
"global": {
"application.def":
".slider/package/HBASE/slider-hbase-app-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
-"java_home": "/usr/jdk64/jdk1.7.0_45",
+"java_home": "/usr/jdk64/jdk1.7.0_67",
"system_configs": "core-site",
"site.global.app_user": "yarn",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0143c210/app-packages/memcached/appConfig-default.json
--
diff --git a/app-packages/memcached/appConfig-default.json
b/app-packages/memcached/appConfig-default.json
index 7b4aa97..b8b4bf1 100644
--- a/app-packages/memcached/appConfig-default.json
+++ b/app-packages/memcached/appConfig-default.json
@@ -4,7 +4,7 @@
},
"global": {
"application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
-"java_home": "/usr/jdk64/jdk1.7.0_45",
+"java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0143c210/app-packages/storm/appConfig-default.json
--
diff --git a/app-packages/storm/appConfig-default.json
b/app-packages/storm/appConfig-default.json
index 2b0eea0..e01ceda 100644
--- a/app-packages/storm/appConfig-default.json
+++
[17/24] git commit: SLIDER-181. Remove REST service from Storm package
SLIDER-181. Remove REST service from Storm package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4d2133fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4d2133fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4d2133fc
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4d2133fc3ae240c33b01d47bd414785b81803d5d
Parents: 0143c21
Author: Sumit Mohanty
Authored: Fri Oct 3 18:34:52 2014 -0700
Committer: Sumit Mohanty
Committed: Fri Oct 3 18:34:52 2014 -0700
--
app-packages/storm/appConfig-default.json| 2 --
app-packages/storm/metainfo.xml | 23 +++
app-packages/storm/package/scripts/params.py | 2 --
app-packages/storm/resources-default.json| 4
4 files changed, 3 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d2133fc/app-packages/storm/appConfig-default.json
--
diff --git a/app-packages/storm/appConfig-default.json
b/app-packages/storm/appConfig-default.json
index e01ceda..dc07ee3 100644
--- a/app-packages/storm/appConfig-default.json
+++ b/app-packages/storm/appConfig-default.json
@@ -16,8 +16,6 @@
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
"site.global.ganglia_server_port": "8668",
-"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
-"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d2133fc/app-packages/storm/metainfo.xml
--
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index bbb9db6..3d1b048 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -30,7 +30,7 @@
org.apache.slider.jmx
-
http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary
+
http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}/api/cluster/summary
org.apache.slider.monitor
@@ -55,8 +55,7 @@
NIMBUS-START
-
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
-
+
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
SUPERVISOR-START
@@ -70,10 +69,6 @@
STORM_UI_SERVER-START
NIMBUS-STARTED
-
-STORM_REST_API-START
-
NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED
-
@@ -111,7 +106,7 @@
STORM_UI_SERVER
MASTER
true
-QuickLinks-org.apache.slider.monitor
+
QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx
true
scripts/ui_server.py
@@ -130,18 +125,6 @@
600
-
-
-STORM_REST_API
-MASTER
-true
-QuickLinks-org.apache.slider.jmx
-
- scripts/rest_api.py
- PYTHON
- 600
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d2133fc/app-packages/storm/package/scripts/params.py
--
diff --git a/app-packages/storm/package/scripts/params.py
b/app-packages/storm/package/scripts/params.py
index 1332b9f..1ccba5e 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,8 +34,6 @@ user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-rest_api_port = config['configurations']['global']['rest_api_port']
-rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
rest_api_conf_file = format("{conf_dir}/config.yaml")
rest_lib_dir = format("{app_root}/external/storm-rest")
storm_bin = format("{app_root}/bin/storm.py")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d2133fc/app-packages/storm/resources-default.json
--
diff --git a/app-packages/storm/resources-default.json
b/app-packages/storm/resources-default.json
index acecd91..31b445e 100644
--- a/app-packages/storm/resources-default.json
+++ b/app-packag
[06/24] git commit: SLIDER-476 make AM queue executor single thread
SLIDER-476 make AM queue executor single thread
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ddde0300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ddde0300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ddde0300
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ddde0300d3f17fbaa4e0dbdd4d820b18dffc99f9
Parents: 1deafee
Author: Steve Loughran
Authored: Thu Oct 2 13:51:52 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 13:51:52 2014 -0700
--
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ddde0300/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 365eeab..24d5eea 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -197,7 +197,6 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
- public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/**
* Singleton of metrics registry
@@ -212,7 +211,6 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
private AMRMClientAsync asyncRMClient;
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-
private RMOperationHandler rmOperationHandler;
private RMOperationHandler providerRMOperationHandler;
@@ -411,11 +409,10 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
SliderUtils.validateSliderServerEnvironment(log);
executorService = new
WorkflowExecutorService("AmExecutor",
-Executors.newCachedThreadPool(
+Executors.newSingleThreadExecutor(
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
-
addService(actionQueues);
//init all child services
super.serviceInit(conf);
[13/24] git commit: SLIDER-484 app-package/hbase not in slider base pom
SLIDER-484 app-package/hbase not in slider base pom
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e631a132
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e631a132
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e631a132
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e631a13233af331803f50f64fd30272affe6f47c
Parents: fef98cd
Author: Steve Loughran
Authored: Fri Oct 3 15:21:00 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:21:00 2014 -0700
--
app-packages/accumulo/pom.xml | 2 +-
app-packages/hbase/pom.xml| 2 +-
pom.xml | 2 ++
3 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index bc4b591..f2f1a35 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -74,7 +74,7 @@
org.apache.maven.plugins
maven-antrun-plugin
-1.7
+${maven-antrun-plugin.version}
copy
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/app-packages/hbase/pom.xml
--
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 519d7c1..be1b858 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -45,7 +45,7 @@
org.apache.maven.plugins
maven-antrun-plugin
-1.7
+${maven-antrun-plugin.version}
copy
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e631a132/pom.xml
--
diff --git a/pom.xml b/pom.xml
index b31d476..6d35407 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,6 +39,7 @@
slider-core
slider-agent
app-packages/accumulo
+app-packages/hbase
slider-assembly
slider-funtest
slider-providers/hbase/slider-hbase-provider
@@ -193,6 +194,7 @@
[3.0.0,)
+1.7
2.4
2.5.2
3.1
[18/24] git commit: SLIDER-181. Remove REST service from Storm package
SLIDER-181. Remove REST service from Storm package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/757f69fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/757f69fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/757f69fb
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 757f69fbf533efb8b7e7b4aa173fb0bde55c0e19
Parents: 4d2133f
Author: Sumit Mohanty
Authored: Sat Oct 4 10:47:21 2014 -0700
Committer: Sumit Mohanty
Committed: Sat Oct 4 10:47:21 2014 -0700
--
app-packages/storm/package/templates/config.yaml.j2 | 9 -
1 file changed, 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/757f69fb/app-packages/storm/package/templates/config.yaml.j2
--
diff --git a/app-packages/storm/package/templates/config.yaml.j2
b/app-packages/storm/package/templates/config.yaml.j2
index 32d2c99..aa4ec46 100644
--- a/app-packages/storm/package/templates/config.yaml.j2
+++ b/app-packages/storm/package/templates/config.yaml.j2
@@ -16,15 +16,6 @@
nimbusHost: {{nimbus_host}}
nimbusPort: {{nimbus_port}}
-# HTTP-specific options.
-http:
-
- # The port on which the HTTP server listens for service requests.
- port: {{rest_api_port}}
-
- # The port on which the HTTP server listens for administrative requests.
- adminPort: {{rest_api_admin_port}}
-
{% if ganglia_installed %}
enableGanglia: {{ganglia_installed}}
[03/24] git commit: ide-tuning
ide-tuning
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6b011d92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6b011d92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6b011d92
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6b011d92500e6460ac0401ccdd522a84824ea304
Parents: 0b1ff8e
Author: Steve Loughran
Authored: Wed Oct 1 15:40:13 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 1 15:40:13 2014 -0700
--
.../apache/slider/providers/agent/AgentProviderService.java| 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6b011d92/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 755b9d8..d7943b2 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -450,7 +450,7 @@ public class AgentProviderService extends
AbstractProviderService implements
}
Map folders = registration.getLogFolders();
- if (folders != null && folders.size() > 0) {
+ if (folders != null && !folders.isEmpty()) {
publishLogFolderPaths(folders, containerId, roleName, hostFqdn);
}
} else {
@@ -465,9 +465,9 @@ public class AgentProviderService extends
AbstractProviderService implements
/**
* Handle heartbeat response from agents
*
- * @param heartBeat
+ * @param heartBeat incoming heartbeat from Agent
*
- * @return
+ * @return response to send back
*/
@Override
public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
[22/24] git commit: SLIDER-181. Remove REST service from Storm win package and explicitly set STORM_LOG_DIR
SLIDER-181. Remove REST service from Storm win package and explicitly set
STORM_LOG_DIR
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1893c7cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1893c7cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1893c7cb
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1893c7cbb39dbbe84c16b8442d99af4ca7acb373
Parents: ed189c3
Author: Sumit Mohanty
Authored: Sun Oct 5 16:41:57 2014 -0700
Committer: Sumit Mohanty
Committed: Sun Oct 5 16:41:57 2014 -0700
--
app-packages/storm-win/appConfig-default.json | 2 --
app-packages/storm-win/metainfo.xml | 12 ++--
app-packages/storm-win/package/scripts/params.py| 2 --
app-packages/storm-win/package/scripts/service.py | 1 +
app-packages/storm-win/package/scripts/status_params.py | 2 --
app-packages/storm-win/package/templates/config.yaml.j2 | 9 -
app-packages/storm/metainfo.xml | 2 +-
7 files changed, 8 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1893c7cb/app-packages/storm-win/appConfig-default.json
--
diff --git a/app-packages/storm-win/appConfig-default.json
b/app-packages/storm-win/appConfig-default.json
index 428dea5..229bc1d 100644
--- a/app-packages/storm-win/appConfig-default.json
+++ b/app-packages/storm-win/appConfig-default.json
@@ -11,8 +11,6 @@
"site.global.app_root":
"${AGENT_WORK_ROOT}/app/install/storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
-"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
-"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1893c7cb/app-packages/storm-win/metainfo.xml
--
diff --git a/app-packages/storm-win/metainfo.xml
b/app-packages/storm-win/metainfo.xml
index 070ce35..ec68ef6 100644
--- a/app-packages/storm-win/metainfo.xml
+++ b/app-packages/storm-win/metainfo.xml
@@ -29,6 +29,10 @@
QuickLinks
+org.apache.slider.jmx
+
http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}/api/v1/cluster/summary
+
+
org.apache.slider.monitor
http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}
@@ -43,7 +47,7 @@
NIMBUS-START
-
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+
SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
@@ -58,10 +62,6 @@
STORM_UI_SERVER-START
NIMBUS-STARTED
-
-STORM_REST_API-START
-
NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED
-
@@ -99,7 +99,7 @@
STORM_UI_SERVER
MASTER
true
-QuickLinks-org.apache.slider.monitor
+
QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx
true
scripts/ui_server.py
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1893c7cb/app-packages/storm-win/package/scripts/params.py
--
diff --git a/app-packages/storm-win/package/scripts/params.py
b/app-packages/storm-win/package/scripts/params.py
index c6ea62d..3a458ea 100644
--- a/app-packages/storm-win/package/scripts/params.py
+++ b/app-packages/storm-win/package/scripts/params.py
@@ -34,8 +34,6 @@ user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-rest_api_port = config['configurations']['global']['rest_api_port']
-rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
rest_api_conf_file = format("{conf_dir}/config.yaml")
rest_lib_dir = format("{app_root}/external/storm-rest")
storm_bin = format("{app_root}/bin/storm.cmd")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1893c7cb/app-packages/storm-win/package/scripts/service.py
--
diff --git a/app-packages/storm-win/package/scripts/service.py
b/app-packages/storm-win/package/scripts/s
[05/24] git commit: SLIDER-470 slider appears to support negative component counts: client side checks
SLIDER-470 slider appears to support negative component counts: client side
checks
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1deafee8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1deafee8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1deafee8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1deafee8401922f7609ce6333d38b3e52a43f06f
Parents: ca9
Author: Steve Loughran
Authored: Thu Oct 2 13:25:08 2014 -0700
Committer: Steve Loughran
Committed: Thu Oct 2 13:25:08 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 38 ++--
.../apache/slider/core/conf/MapOperations.java | 16 +++--
.../providers/AbstractClientProvider.java | 9 ++---
.../apache/slider/providers/ProviderCore.java | 5 +++
.../slideram/SliderAMClientProvider.java| 28 +++
.../slider/server/appmaster/state/AppState.java | 5 ++-
.../server/appmaster/state/RoleStatus.java | 20 +++
.../standalone/TestBuildStandaloneAM.groovy | 25 +
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
9 files changed, 120 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1deafee8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 06c37ba..05ef532 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -842,17 +842,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// make any substitutions needed at this stage
replaceTokens(appConf.getConfTree(), getUsername(), clustername);
-// provider to validate what there is
-try {
- sliderAM.validateInstanceDefinition(builder.getInstanceDescription());
- provider.validateInstanceDefinition(builder.getInstanceDescription());
-} catch (SliderException e) {
- //problem, reject it
- log.info("Error {} validating application instance definition ",
e.toString());
- log.debug("Error validating application instance definition ", e);
- log.info(instanceDefinition.toString());
- throw e;
-}
+// providers to validate what there is
+AggregateConf instanceDescription = builder.getInstanceDescription();
+validateInstanceDefinition(sliderAM, instanceDescription);
+validateInstanceDefinition(provider, instanceDescription);
try {
persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
@@ -2060,6 +2053,10 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
clusterDirectory);
+SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+// provider to validate what there is
+SliderAMClientProvider provider = sliderAM;
+validateInstanceDefinition(provider, instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
@@ -2107,6 +2104,25 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+ /**
+ * Validate an instance definition against a provider.
+ * @param provider the provider performing the validation
+ * @param instanceDefinition the instance definition
+ * @throws SliderException if invalid.
+ */
+ protected void validateInstanceDefinition(AbstractClientProvider provider,
+ AggregateConf instanceDefinition) throws SliderException {
+try {
+ provider.validateInstanceDefinition(instanceDefinition);
+} catch (SliderException e) {
+ //problem, reject it
+ log.info("Error {} validating application instance definition ", e);
+ log.debug("Error validating application instance definition ", e);
+ log.info(instanceDefinition.toString());
+ throw e;
+}
+ }
+
/**
* Load the persistent cluster description
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1deafee8/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 4b1b44f..bba3ee2 100644
--- a/slider-core/src/main/java
[02/24] git commit: SLIDER-455 TestStandaloneAMMonkeyRestart test failure
SLIDER-455 TestStandaloneAMMonkeyRestart test failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b1ff8e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b1ff8e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b1ff8e1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0b1ff8e142218a60843697681a7d1419a324f7f8
Parents: 2bbaafd
Author: Steve Loughran
Authored: Wed Oct 1 14:50:43 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 1 14:50:43 2014 -0700
--
.../standalone/TestStandaloneAMMonkeyRestart.groovy | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b1ff8e1/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
--
diff --git
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
index b37b9a6..5136092 100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -59,13 +59,11 @@ class TestStandaloneAMMonkeyRestart extends
AgentMiniClusterTestBase {
ApplicationReport report
report = waitForClusterLive(sliderClient, 3)
describe "Waiting for the cluster to fail"
-sleep(4)
-// end of process
-report = sliderClient.applicationReport
-log.info(report.diagnostics)
-assert report.currentApplicationAttemptId.attemptId == threshold
-assert YarnApplicationState.FAILED == report.yarnApplicationState
-assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+def finishedReport = waitForAppToFinish(sliderClient, 9)
+log.info(finishedReport.diagnostics)
+assert finishedReport.currentApplicationAttemptId.attemptId == threshold
+assert YarnApplicationState.FAILED == finishedReport.yarnApplicationState
+assert FinalApplicationStatus.FAILED ==
finishedReport.finalApplicationStatus
}
/**
[12/24] git commit: SLIDER-467 final status == SUCCEEDED on shut down app
SLIDER-467 final status == SUCCEEDED on shut down app
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1a3e45bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1a3e45bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1a3e45bc
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 1a3e45bce82dd36a98e0228648e358f3ea6e0b49
Parents: 2051dd6
Author: Steve Loughran
Authored: Fri Oct 3 15:08:52 2014 -0700
Committer: Steve Loughran
Committed: Fri Oct 3 15:08:52 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 24 ++-
.../TriggerClusterTeardownException.java| 16 +-
.../apache/slider/server/appmaster/AMUtils.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 191 +++
.../appmaster/actions/ActionStopSlider.java | 98 +-
.../server/appmaster/actions/QueueExecutor.java | 8 +
.../server/appmaster/actions/QueueService.java | 22 ++-
.../slider/server/appmaster/state/AppState.java | 8 +-
.../agent/actions/TestActionStatus.groovy | 7 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 13 +-
.../standalone/TestStandaloneAMDestroy.groovy | 17 +-
.../standalone/TestStandaloneAMRestart.groovy | 29 ++-
.../TestMockAppStateContainerFailure.groovy | 1 -
.../slider/test/YarnMiniClusterTestBase.groovy | 24 ++-
.../accumulo/live/TestAccFreezeThaw.groovy | 7 +-
15 files changed, 337 insertions(+), 131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a3e45bc/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dc81f98..d1a1a7b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1545,7 +1545,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
/**
- * Kill the submitted application by sending a call to the ASM
+ * Kill the submitted application via YARN
* @throws YarnException
* @throws IOException
*/
@@ -1875,7 +1875,6 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
LaunchedApplication application = new LaunchedApplication(yarnClient, app);
applicationId = application.getApplicationId();
-
if (forcekill) {
//escalating to forced kill
application.kill("Forced stop of " + clustername +
@@ -2423,7 +2422,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
* @throws IOException
* Network or other problems
*/
- private int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
try {
if (diagnosticArgs.client) {
actionDiagnosticClient();
@@ -2465,7 +2464,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// we are catching exceptions here because those are
indication of
// validation result, and we need to print them here
log.error("validation of slider-client.xml fails
because: "
- + e.toString());
+ + e.toString(), e);
return;
}
SliderClusterOperations clusterOperations =
createClusterOperations(clusterName);
@@ -2489,19 +2488,24 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
}
try {
SliderUtils.validateHDFSFile(sliderFileSystem,
imagePath);
- log.info("Slider agent tarball is properly
installed");
+ log.info("Slider agent package is properly
installed");
+ } catch (FileNotFoundException e) {
+ log.error("can not find agent package: {}", e);
+ return;
} catch (IOException e) {
- log.error("can not find or open agent tar ball:
" + e.toString());
+ log.error("can not open agent package: {}", e,
e);
return;
}
String pkgTarballPath =
instanceDefin
svn commit: r1629802 - /incubator/slider/site/trunk/content/developing/windows.md
Author: stevel Date: Tue Oct 7 04:39:33 2014 New Revision: 1629802 URL: http://svn.apache.org/r1629802 Log: more on windows setup Modified: incubator/slider/site/trunk/content/developing/windows.md Modified: incubator/slider/site/trunk/content/developing/windows.md URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/windows.md?rev=1629802&r1=1629801&r2=1629802&view=diff == --- incubator/slider/site/trunk/content/developing/windows.md (original) +++ incubator/slider/site/trunk/content/developing/windows.md Tue Oct 7 04:39:33 2014 @@ -66,7 +66,8 @@ even just for viewing logs & test result 1. ...JEdit is fairly nimble too, though not as lightweight. 1. Chrome can be used a browser instead of IE if you want —but you should still view all web pages in IE to verify they look OK. - +1. [Sysinternals](http://technet.microsoft.com/en-us/sysinternals/bb545021.aspx). +Their `tcpview` tool is ideal for seeing what ports Hadoop services are listening on. ### Knowledge @@ -185,12 +186,13 @@ in the root directory of the hadoop repo (use two commands as sometimes clean can fail with some recursive directory-in-use exception) - pushd hadoop-dist\target tar -xvzf hadoop-%HADOOP_VERSION%.tar.gz rm -rf c:\bin\hadoop cp -r hadoop-%HADOOP_VERSION% c:\bin\hadoop popd + + Hadoop is now installed into `c:\bin\%HADOOP_VERSION%`
svn commit: r1629803 - /incubator/slider/site/trunk/content/design/registry/references.md
Author: stevel Date: Tue Oct 7 04:39:56 2014 New Revision: 1629803 URL: http://svn.apache.org/r1629803 Log: SLIDER-149 updated registry references Modified: incubator/slider/site/trunk/content/design/registry/references.md Modified: incubator/slider/site/trunk/content/design/registry/references.md URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/design/registry/references.md?rev=1629803&r1=1629802&r2=1629803&view=diff == --- incubator/slider/site/trunk/content/design/registry/references.md (original) +++ incubator/slider/site/trunk/content/design/registry/references.md Tue Oct 7 04:39:56 2014 @@ -28,11 +28,12 @@ The first documented directory service; * **[Das02]** [*SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol*](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf) P2P gossip-style data sharing protocol with random liveness probes to address scalable liveness checking. Ceph uses similar liveness checking. -* **[Marti02]** Marti S. and Krishnam V., [*Carmen: A Dynamic Service Discovery Architecture*](http://www.hpl.hp.com/techreports/2002/HPL-2002-257), - * **[Lampson86]** Lampson, B. [*Designing a Global Naming Service*](http://research.microsoft.com/en-us/um/people/blampson/36-GlobalNames/Acrobat.pdf). DEC. Distributed; includes an update protocol and the ability to add links to other parts of the tree. Also refers to [*Xerox Clearinghouse*](http://bitsavers.informatik.uni-stuttgart.de/pdf/xerox/parc/techReports/OPD-T8103_The_Clearinghouse.pdf), which apparently shipped. + +* **[Marti02]** Marti S. and Krishnam V., [*Carmen: A Dynamic Service Discovery Architecture*](http://www.hpl.hp.com/techreports/2002/HPL-2002-257), + * **[Mockapetris88]** Mockapetris, P. [*Development of the domain name system*](http://bnrg.eecs.berkeley.edu/~randy/Courses/CS268.F08/papers/31_dns.pdf). The history of DNS * **[Schroeder84]** Schroeder, M.D. et al, [*Experience with Grapevine: The Growth of a Distributed System*](http://research.microsoft.com/apps/pubs/default.aspx?id=61509). Xerox. @@ -42,7 +43,7 @@ Writeup of the experiences of using grap Grandest P2P management framework to date; the work that earned Werner Vogel his CTO position at Amazon. * **[van Steen86]** van Steen, M. et al, [*A Scalable Location Service for Distributed Objects*](http://www.cs.vu.nl/~ast/publications/asci-1996a.pdf). -Vrije Universiteit, Amsterdam. Probably the first Object Request Broker +Vrije Universiteit, Amsterdam. Probably the first Object Request Broker. ORBs may be out of favour, with CORBA being viewed with disdain, but the goals of ORBs were not all wrong. One of the aspects of the early ORB-deployed applications wasn't the internal implementation of back-end web applications, it was to implement distributed client-server applications where the back end was the model, the front end the view; ORBs would bond the two even as the data moved around. Ignoring the implementation details of how that client-side code got there, it is a cleaner model than many web applications have today.
[1/5] git commit: SLIDER-365 slider "resolve" command to retrieve service record
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry c51f2f0ea ->
bfbc67aff
SLIDER-365 slider "resolve" command to retrieve service record
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d084e9e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d084e9e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d084e9e8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d084e9e86f145655af3eafe2da76db08269db54a
Parents: c51f2f0
Author: Steve Loughran
Authored: Mon Oct 6 13:53:49 2014 -0700
Committer: Steve Loughran
Committed: Mon Oct 6 13:53:49 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 56 ++--
.../common/params/ActionRegistryArgs.java | 9 +-
.../slider/common/params/ActionResolveArgs.java | 92
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 65 --
.../slider/common/params/SliderActions.java | 1 +
6 files changed, 183 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 26e5707..a5bbccc 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -72,6 +72,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
@@ -332,9 +333,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
+} else if (ACTION_DIAGNOSTIC.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
} else if (ACTION_GETCONF.equals(action)) {
@@ -351,8 +354,9 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
} else if (ACTION_LIST.equals(action)) {
exitCode = actionList(clusterName);
} else if (ACTION_REGISTRY.equals(action)) {
- exitCode = actionRegistry(
- serviceArgs.getActionRegistryArgs());
+ exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
+} else if (ACTION_RESOLVE.equals(action)) {
+ exitCode = actionResolve(serviceArgs.getActionResolveArgs());
} else if (ACTION_STATUS.equals(action)) {
exitCode = actionStatus(clusterName,
serviceArgs.getActionStatusArgs());
@@ -362,8 +366,6 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
} else if (ACTION_VERSION.equals(action)) {
exitCode = actionVersion();
-} else if (ACTION_DIAGNOSTIC.equals(action)) {
-exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
"Unimplemented: " + action);
@@ -2387,13 +2389,51 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
/**
* Registry operation
*
+ * @param args registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ public int actionResolve(ActionResolveArgs args) throws
+ YarnException,
+ IOException {
+// as this is also a test entry point, validate
+// the arguments
+args.validate();
+RegistryOperations operations = getRegistryOperations();
+String serviceclassPath = args.path;
+Collection serviceRecords;
+
[4/5] git commit: SLIDER-149 static sample.json resync'd with current schema
SLIDER-149 static sample.json resync'd with current schema
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29c98240
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29c98240
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29c98240
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 29c982403ea6ae3f0c070d5a9e24a27608127e80
Parents: 330579e
Author: Steve Loughran
Authored: Tue Oct 7 08:56:48 2014 -0700
Committer: Steve Loughran
Committed: Tue Oct 7 08:56:48 2014 -0700
--
.../slider/server/appmaster/web/rest/registry/sample.json | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/29c98240/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
--
diff --git
a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
index b2d59fd..4d2c6db 100644
---
a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
+++
b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
@@ -1,9 +1,8 @@
{
"nodes":
["/users/example/services/org-apache-slider/test-registry-rest-resources/components"],
"service": {
- "registrationTime": 0,
"description": "Slider Application Master",
- "yarn_id": "application_1411664296263_0001",
- "yarn_persistence": 1,
+ "yarn:id": "application_1411664296263_0001",
+ "yarn:persistence": 1,
"external": [
{
"api": "org.apache.slider.appmaster",
[5/5] git commit: SLIDER-149/YARN-913 move registry package to org.apache.hadoop.registry
SLIDER-149/YARN-913 move registry package to org.apache.hadoop.registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfbc67af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfbc67af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfbc67af
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bfbc67affeb711bf6d20aa7ca76793ea840660b9
Parents: 29c9824
Author: Steve Loughran
Authored: Tue Oct 7 11:45:26 2014 -0700
Committer: Steve Loughran
Committed: Tue Oct 7 11:45:26 2014 -0700
--
.../funtest/accumulo/AccumuloBasicIT.groovy | 6 +++---
.../org/apache/slider/client/SliderClient.java| 18 +-
.../core/registry/retrieve/RegistryRetriever.java | 10 --
.../slider/providers/AbstractProviderService.java | 10 +-
.../apache/slider/providers/ProviderService.java | 2 +-
.../providers/agent/AgentProviderService.java | 7 +++
.../slideram/SliderAMProviderService.java | 6 ++
.../slider/server/appmaster/SliderAppMaster.java | 14 +++---
.../server/appmaster/state/RoleInstance.java | 6 +++---
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../slider/server/appmaster/web/WebAppApi.java| 2 +-
.../server/appmaster/web/WebAppApiImpl.java | 2 +-
.../web/rest/registry/PathEntryResource.java | 2 +-
.../web/rest/registry/RegistryResource.java | 10 +-
.../utility/AbstractSliderLaunchedService.java| 6 +++---
.../YarnRegistryViewForProviders.java | 10 +-
.../TestStandaloneYarnRegistryAM.groovy | 14 +++---
.../model/mock/MockProviderService.groovy | 2 +-
.../web/rest/registry/PathEntryMarshalling.groovy | 2 +-
.../registry/TestRegistryRestMarshalling.groovy | 2 +-
.../registry/TestRegistryRestResources.groovy | 6 +++---
.../org/apache/slider/test/MicroZKCluster.groovy | 6 +++---
.../org/apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnZKMiniClusterTestBase.groovy | 4 +---
.../providers/hbase/HBaseProviderService.java | 8
.../hbase/minicluster/live/TestHBaseMaster.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 8
27 files changed, 83 insertions(+), 90 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 2f6674e..bb9abba 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.ProviderUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.alias.CredentialProvider
import org.apache.hadoop.security.alias.CredentialProviderFactory
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
@@ -39,8 +39,8 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
-import static
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.currentUser
-import static
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.servicePath
+import static
org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser
+import static
org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index b3365a7..185e77d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -41,15 +41,15 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import
[2/5] git commit: SLIDER-149 in sync with latest YARN-913 changes
SLIDER-149 in sync with latest YARN-913 changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27381a6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27381a6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27381a6d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 27381a6dc29589a06d98be43901f175230d54198
Parents: d084e9e
Author: Steve Loughran
Authored: Mon Oct 6 21:20:02 2014 -0700
Committer: Steve Loughran
Committed: Mon Oct 6 21:20:02 2014 -0700
--
.../funtest/accumulo/AccumuloBasicIT.groovy | 10 ++-
.../org/apache/slider/client/SliderClient.java | 64
.../server/appmaster/SliderAppMaster.java | 18 +++---
.../YarnRegistryViewForProviders.java | 8 +--
.../TestStandaloneYarnRegistryAM.groovy | 19 +++---
.../registry/TestRegistryRestMarshalling.groovy | 5 +-
.../registry/TestRegistryRestResources.groovy | 8 ++-
.../apache/slider/test/MicroZKCluster.groovy| 2 +-
.../providers/hbase/HBaseProviderService.java | 9 ++-
.../minicluster/live/TestHBaseMaster.groovy | 3 +-
10 files changed, 86 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
--
diff --git
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index fd8af9a..2f6674e 100644
---
a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++
b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -30,7 +30,6 @@ import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.persist.ConfTreeSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfiguration
@@ -40,6 +39,9 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
+import static
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.currentUser
+import static
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.servicePath
+
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
protected static final String PROVIDER_PROPERTY = "site.accumulo-site." +
@@ -170,8 +172,10 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase
{
Exception caught;
while (true) {
try {
-ServiceRecord instance =
- sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName)
+String path = servicePath(currentUser(),
+SliderKeys.APP_TYPE,
+clusterName);
+ServiceRecord instance = sliderClient.resolve(path)
RegistryRetriever retriever = new RegistryRetriever(instance)
PublishedConfiguration configuration = retriever.retrieveConfiguration(
retriever.getConfigurations(true), "quicklinks", true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index a5bbccc..602d6cf 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -48,7 +48,6 @@ import static
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*;
import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -127,15 +126,11 @@ import
org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
-import org.codehaus.jackson.map.DeserializationConfig;
[3/5] git commit: SLIDER-149/YARN-913 persistence policy set via set() method
SLIDER-149/YARN-913 persistence policy set via set() method
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/330579e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/330579e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/330579e4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 330579e47f8bc3cd520d23a0e3c4bcae10d2ee45
Parents: 27381a6
Author: Steve Loughran
Authored: Mon Oct 6 21:42:39 2014 -0700
Committer: Steve Loughran
Committed: Mon Oct 6 21:42:39 2014 -0700
--
.../java/org/apache/slider/client/SliderClient.java | 5 +++--
.../slider/server/appmaster/SliderAppMaster.java| 16 ++--
.../standalone/TestStandaloneYarnRegistryAM.groovy | 7 ---
.../providers/hbase/HBaseProviderService.java | 9 ++---
4 files changed, 23 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 602d6cf..b3365a7 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -49,6 +49,7 @@ import
org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import
org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
@@ -2789,9 +2790,9 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.getYarn_id());
+ log.info("{}", instance.get(YarnRegistryAttributes.YARN_ID, ""));
} else {
- log.info("{}: ", instance.getYarn_id());
+ log.info("{}: ", instance.get(YarnRegistryAttributes.YARN_ID, ""));
logEndpoints(instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 6f7275c..5f7ebe6 100644
---
a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++
b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -59,6 +59,7 @@ import
org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import
org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
import
org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
@@ -931,8 +932,9 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
-serviceRecord.putYarn_id(appid.toString());
-serviceRecord.putYarn_persistence(PersistencePolicies.APPLICATION);
+serviceRecord.set(YarnRegistryAttributes.YARN_ID, appid.toString());
+serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+PersistencePolicies.APPLICATION);
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -964,8 +966,9 @@ public class SliderAppMaster extends
AbstractSliderLaunchedService
// and a shorter lived binding to the app
String attempt = appAttemptID.toString();
ServiceRecord attemptRecord = new ServiceRecord(serviceRecord);
-attemptRecord.putYarn_id(attempt);
-attemptRecord.putYarn_persistence(PersistencePolicies.APPLICATION_ATTEMPT);
+attemptRecord.set(YarnRegistryAttributes.YARN_ID, attempt);
+attemptRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+Persistence
[2/8] git commit: SLIDER-474 enable keytab-based security for AM
SLIDER-474 enable keytab-based security for AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2359c6dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2359c6dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2359c6dd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2359c6ddbaab56c84a4c901b09c025593eee08ef
Parents: 80e8df0
Author: Jon Maron
Authored: Mon Oct 6 15:50:36 2014 -0400
Committer: Jon Maron
Committed: Mon Oct 6 15:50:36 2014 -0400
--
.../org/apache/slider/common/SliderKeys.java| 4 +
.../slider/common/tools/CoreFileSystem.java | 15 ++
.../providers/agent/AgentProviderService.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 161 ++-
.../security/SecurityConfiguration.java | 201 +++
.../security/SecurityConfigurationTest.groovy | 159 +++
6 files changed, 520 insertions(+), 47 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index ddb9ee0..4348fb0 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -86,6 +86,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
+ String KEYTAB_DIR = "keytabs";
/**
* Filename pattern is required to save in strict temporal order.
@@ -170,6 +171,9 @@ public interface SliderKeys extends SliderXmlConfKeys {
String PASSPHRASE = "DEV";
String PASS_LEN = "50";
String KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ String AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
/**
* Python specific
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 955d991..b6e6ecf 100644
---
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -134,6 +134,21 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabName, String applicationName) {
+Preconditions.checkNotNull(applicationName);
+Path basePath = getBaseApplicationPath();
+Path baseKeytabDir = new Path(basePath, SliderKeys.KEYTAB_DIR);
+Path appKeytabDir = new Path(baseKeytabDir, applicationName);
+return keytabName == null ? appKeytabDir :
+new Path(appKeytabDir, keytabName);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 67a268e..88c8709 100644
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -325,6 +326,32 @@ public class AgentProviderService extends
AbstractProviderService impl
[4/8] git commit: SLIDER-341. Add a window based failure count for auto-start to limit indefinite attempt
SLIDER-341. Add a window based failure count for auto-start to limit indefinite
attempt
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/961e1704
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/961e1704
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/961e1704
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 961e17043f589c4096dce896d64a582dd000dd4e
Parents: d8b36ca
Author: Sumit Mohanty
Authored: Mon Oct 6 20:47:44 2014 -0700
Committer: Sumit Mohanty
Committed: Mon Oct 6 20:47:44 2014 -0700
--
slider-agent/conf/agent.ini | 1 +
.../src/main/python/agent/AgentConfig.py| 14 +
.../src/main/python/agent/Controller.py | 35 ++-
.../src/test/python/agent/TestController.py | 63
slider-agent/src/test/python/agent/TestMain.py | 37
5 files changed, 148 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/conf/agent.ini
--
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
index 7b9d57d..48113e3 100644
--- a/slider-agent/conf/agent.ini
+++ b/slider-agent/conf/agent.ini
@@ -43,6 +43,7 @@ log_level=INFO
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/main/python/agent/AgentConfig.py
--
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py
b/slider-agent/src/main/python/agent/AgentConfig.py
index e45ba23..86925b1 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -61,6 +61,7 @@ log_level=INFO
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
keysdir=security/keys
@@ -109,6 +110,8 @@ class AgentConfig:
# agent version file
VERSION_FILE = "version_file"
+ AUTO_RESTART = "auto_restart"
+
FOLDER_MAPPING = {
APP_PACKAGE_DIR: "WORK",
APP_INSTALL_DIR: "WORK",
@@ -164,6 +167,17 @@ class AgentConfig:
return ""
return command
+ # return max, window - max failures within window minutes
+ def getErrorWindow(self):
+window = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+if window != None:
+ parts = window.split(',')
+ if len(parts) == 2:
+if parts[0].isdigit() and parts[1].isdigit():
+ return (int(parts[0]), int(parts[1]))
+ pass
+return (0, 0)
+
def set(self, category, name, value):
global config
return config.set(category, name, value)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/main/python/agent/Controller.py
--
diff --git a/slider-agent/src/main/python/agent/Controller.py
b/slider-agent/src/main/python/agent/Controller.py
index 11db21c..77f932c 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -27,6 +27,7 @@ import time
import threading
import urllib2
import pprint
+import math
from random import randint
from AgentConfig import AgentConfig
@@ -86,7 +87,8 @@ class Controller(threading.Thread):
self.statusCommand = None
self.failureCount = 0
self.heartBeatRetryCount = 0
-self.autoRestart = False
+self.autoRestartFailures = 0
+self.autoRestartTrackingSince = 0
def __del__(self):
@@ -275,7 +277,7 @@ class Controller(threading.Thread):
stored_command =
self.actionQueue.customServiceOrchestrator.stored_command
if len(stored_command) > 0:
auto_start_command = self.create_start_command(stored_command)
-if auto_start_command:
+if auto_start_command and self.shouldAutoRestart():
logger.info("Automatically adding a start command.")
logger.debug("Auto start command: " +
pprint.pformat(auto_start_command))
self.updateStateBasedOnCommand([auto_start_command], False)
@@ -486,6 +488,35 @@ class Controller(threading.Thread):
return {'exitstatus': 1, 'log': err_msg}
+ # Basic window that only counts failures till the window duration expires
+ def shouldAutoRestart(self):
+max, window = self.config.getErrorWindow()
+if max <= 0 or window <= 0:
+ return True
+
+seconds_now = time.time()
+if self.autoRestartTrackingSince == 0:
+ self.autoRestartTrackingSince = seconds_now
+ self.autoRestartFailures = 1
+ return True
+
+
[6/8] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7f7bb5b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7f7bb5b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7f7bb5b8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 7f7bb5b832a09d3420cc8a3c801b50a5f923e2b5 Parents: bfbc67a 0419a5a Author: Steve Loughran Authored: Tue Oct 7 15:18:04 2014 -0700 Committer: Steve Loughran Committed: Tue Oct 7 15:18:04 2014 -0700 -- app-packages/accumulo/appConfig-default.json| 2 +- app-packages/accumulo/pom.xml | 1 + pom.xml | 6 + slider-agent/conf/agent.ini | 1 + .../src/main/python/agent/AgentConfig.py| 14 ++ .../src/main/python/agent/Controller.py | 35 +++- .../src/test/python/agent/TestController.py | 63 ++ slider-agent/src/test/python/agent/TestMain.py | 37 .../org/apache/slider/common/SliderKeys.java| 4 + .../slider/common/tools/CoreFileSystem.java | 15 ++ .../providers/agent/AgentProviderService.java | 27 +++ .../server/appmaster/SliderAppMaster.java | 161 ++- .../security/SecurityConfiguration.java | 201 +++ .../security/SecurityConfigurationTest.groovy | 159 +++ .../AppsThroughAgentQueueAndLabelsIT.groovy | 26 +++ 15 files changed, 702 insertions(+), 50 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/pom.xml -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java --
[7/8] git commit: minor cleanup of utility classes, mostly formatting
minor cleanup of utility classes, mostly formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c09205f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c09205f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c09205f6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c09205f66725438854456379664fae8a0fa298ab
Parents: 7f7bb5b
Author: Steve Loughran
Authored: Tue Oct 7 15:41:14 2014 -0700
Committer: Steve Loughran
Committed: Tue Oct 7 15:41:44 2014 -0700
--
.../slider/common/tools/CoreFileSystem.java | 31 +++-
.../apache/slider/common/tools/SliderUtils.java | 14 -
2 files changed, 17 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index b6e6ecf..2ea371b 100644
---
a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++
b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -158,11 +158,10 @@ public class CoreFileSystem {
* @throws java.io.IOException trouble
* @throws SliderException slider-specific exceptions
*/
- public Path createClusterDirectories(String clustername, Configuration conf)
throws
-
IOException,
- SliderException {
-
-
+ public Path createClusterDirectories(String clustername, Configuration conf)
+ throws IOException, SliderException {
+
+
Path clusterDirectory = buildClusterDirPath(clustername);
InstancePaths instancePaths = new InstancePaths(clusterDirectory);
createClusterDirectories(instancePaths);
@@ -204,8 +203,8 @@ public class CoreFileSystem {
*
* @param dir directory
* @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws org.apache.slider.core.exceptions.BadClusterStateException any
cluster state problem
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
*/
public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
IOException,
@@ -572,7 +571,7 @@ public class CoreFileSystem {
*
* @param clustername name of the cluster
* @return the path to the spec.
- * @throws IOException IO problems
+ * @throws IOException IO problems
* @throws SliderException if the path isn't there
*/
public Path locateInstanceDefinition(String clustername) throws IOException,
@@ -591,23 +590,15 @@ public class CoreFileSystem {
* @throws IOException IO problems
* @throws SliderException if the cluster specification is not present
*/
- public void verifyClusterSpecExists(String clustername,
- Path clusterSpecPath) throws
- IOException,
+ public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
+ throws IOException,
SliderException {
if (!fileSystem.isFile(clusterSpecPath)) {
log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(clustername
-+
-"\n
(definition not found at "
-+
-
clusterSpecPath);
+ throw UnknownApplicationInstanceException.unknownInstance(
+ clustername + "\n (definition not found at " + clusterSpecPath);
}
}
-
- public Path fileToPath(File file) {
-return new Path(file.getAbsoluteFile().toURI());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 5313ab9..9c93753 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/
[5/8] git commit: SLIDER-480. Add a fun test to ensure that containers are restarted on nodes with correct labels
SLIDER-480. Add a fun test to ensure that containers are restarted on nodes with correct labels Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0419a5a9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0419a5a9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0419a5a9 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 0419a5a90afea0e3e7eab0f2d5ea09ab5aad7f89 Parents: 961e170 Author: Sumit Mohanty Authored: Mon Oct 6 21:12:36 2014 -0700 Committer: Sumit Mohanty Committed: Mon Oct 6 21:12:36 2014 -0700 -- .../AppsThroughAgentQueueAndLabelsIT.groovy | 26 1 file changed, 26 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0419a5a9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy -- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy index f3f6612..a3b0ccb 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy @@ -29,6 +29,32 @@ import org.apache.slider.funtest.framework.SliderShell import org.junit.After import org.junit.Test +/** + * SETUP FOR THE TEST + * Create valid labels, red and blue [yarn rmadmin -addLabels red,blue] + * Add nodes with label [yarn rmadmin -setNodeToLabels host1:blue] + * Perform refresh queue [yarn rmadmin -refreshQueues] + * + * Create a queue with access to labels - these are changes to capacity scheduler configuration + * Add a queue in addition to default + * yarn.scheduler.capacity.root.queues=default,labeled + * Provide capacity, take out from default + * yarn.scheduler.capacity.root.labeled.capacity=80 + * yarn.scheduler.capacity.root.default.capacity=20 + * Provide standard queue specs + * yarn.scheduler.capacity.root.labeled.state=RUNNING + * yarn.scheduler.capacity.root.labeled.maximum-capacity=80 + * Have queue access the label + * yarn.scheduler.capacity.root.labeled.labels=red,blue + * + * After specifying the new configuration call refresh [yarn rmadmin -refreshQueues] + * + * See resources_queue_labels.json for label configuration required for the test + * Label expression for slider-appmaster is also the default for all containers + * if they do not specify own label expressions + * "yarn.label.expression":"red" + * + */ @CompileStatic @Slf4j public class AppsThroughAgentQueueAndLabelsIT extends AgentCommandTestBase
[3/8] git commit: SLIDER-489 set accumulo app package appConfig java home in a variable
SLIDER-489 set accumulo app package appConfig java home in a variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d8b36caf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d8b36caf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d8b36caf
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d8b36cafe8966f9b7cd655e3ead535559403abb6
Parents: 2359c6d
Author: Billie Rinaldi
Authored: Mon Oct 6 17:10:37 2014 -0700
Committer: Billie Rinaldi
Committed: Mon Oct 6 17:10:37 2014 -0700
--
app-packages/accumulo/appConfig-default.json | 2 +-
app-packages/accumulo/pom.xml| 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8b36caf/app-packages/accumulo/appConfig-default.json
--
diff --git a/app-packages/accumulo/appConfig-default.json
b/app-packages/accumulo/appConfig-default.json
index 62050af..9e04564 100644
--- a/app-packages/accumulo/appConfig-default.json
+++ b/app-packages/accumulo/appConfig-default.json
@@ -4,7 +4,7 @@
},
"global": {
"application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
-"java_home": "/usr/lib/jvm/java",
+"java_home": "${app.java.home}",
"site.global.app_user": "${app.user}",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8b36caf/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index f2f1a35..4cd5cca 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -41,6 +41,7 @@
/usr/lib
${hadoop.dir}
+${java.home}
yarn
hadoop
[8/8] git commit: SLIDER-365 "slider resolve" command
SLIDER-365 "slider resolve" command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/032dacf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/032dacf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/032dacf4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 032dacf4cf1757302fffc59782572f574d185cdc
Parents: c09205f
Author: Steve Loughran
Authored: Tue Oct 7 17:03:21 2014 -0700
Committer: Steve Loughran
Committed: Tue Oct 7 17:03:21 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 301 +++
.../apache/slider/client/SliderClientAPI.java | 249 +++
.../slider/common/params/ActionResolveArgs.java | 5 -
.../apache/slider/common/params/ClientArgs.java | 4 +-
.../apache/slider/common/tools/SliderUtils.java | 30 +-
.../TestStandaloneYarnRegistryAM.groovy | 40 ++-
6 files changed, 418 insertions(+), 211 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 185e77d..c5022ee 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -140,6 +141,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -159,7 +161,7 @@ import static
org.apache.slider.common.params.SliderActions.*;
*/
public class SliderClient extends AbstractSliderLaunchedService implements
RunService,
-SliderExitCodes, SliderKeys, ErrorStrings {
+SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
private static final Logger log =
LoggerFactory.getLogger(SliderClient.class);
private ClientArgs serviceArgs;
@@ -197,6 +199,16 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
new YarnConfiguration();
}
+ /**
+ * This is called Before serviceInit is called
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the post-binding configuration to pass to the init()
+ * operation.
+ * @throws Exception
+ */
@Override
public Configuration bindArgs(Configuration config, String... args) throws
Exception {
config = super.bindArgs(config, args);
@@ -218,9 +230,11 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-
+AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+if (coreAction.getHadoopServicesRequired()) {
+ initHadoopBinding();
+}
super.serviceInit(conf);
-
}
/**
@@ -308,11 +322,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
-
-AbstractActionArgs coreAction = serviceArgs.getCoreAction();
-if (coreAction.getHadoopServicesRequired()) {
- initHadoopBinding();
-}
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -358,9 +368,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionStatusArgs());
} else if (ACTION_UPDATE.equals(action)) {
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
-
} else if (ACTION_VERSION.equals(action)) {
-
exitCode = actionVersion();
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
@@ -490,11 +498,7 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return client;
}
- /**
- * Destroy a cluster. T
[1/8] git commit: SLIDER-488 excluded transitive commons-codec dependency from commons-httpclient
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry bfbc67aff ->
032dacf4c
SLIDER-488 excluded transitive commons-codec dependency from commons-httpclient
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80e8df0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80e8df0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80e8df0c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 80e8df0c66e41a06943a5e7555062ce05a2e0d22
Parents: 70b4b75
Author: Billie Rinaldi
Authored: Mon Oct 6 09:21:32 2014 -0700
Committer: Billie Rinaldi
Committed: Mon Oct 6 09:21:32 2014 -0700
--
pom.xml | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80e8df0c/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 6d35407..4df4971 100644
--- a/pom.xml
+++ b/pom.xml
@@ -600,6 +600,12 @@
commons-httpclient
commons-httpclient
${httpclient.version}
+
+
+commons-codec
+commons-codec
+
+
git commit: SLIDER-408 slider getconf command should be removed (yu liu)
Repository: incubator-slider
Updated Branches:
refs/heads/develop 6b19cb81b -> e37572697
SLIDER-408 slider getconf command should be removed (yu liu)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e3757269
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e3757269
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e3757269
Branch: refs/heads/develop
Commit: e37572697cb3018a9065099ca13f0032dfa70ce4
Parents: 6b19cb8
Author: Steve Loughran
Authored: Wed Oct 8 11:39:15 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 8 11:39:15 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 97
.../slider/common/params/ActionGetConfArgs.java | 51 --
.../apache/slider/common/params/ClientArgs.java | 9 --
.../slider/common/params/SliderActions.java | 1 -
.../slider/client/TestCommonArgParsing.groovy | 31 ---
.../funtest/framework/CommandTestBase.groovy| 13 ---
.../commands/UnknownClusterOperationsIT.groovy | 6 --
.../lifecycle/AgentClusterLifecycleIT.groovy| 2 -
.../funtest/HBaseClusterLifecycleIT.groovy | 5 +-
9 files changed, 4 insertions(+), 211 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3757269/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git
a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2b76faf..8d2bcfd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -58,7 +58,6 @@ import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionStatusArgs;
@@ -325,8 +324,6 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
-} else if (ACTION_GETCONF.equals(action)) {
- exitCode = actionGetConf(clusterName,
serviceArgs.getActionGetConfArgs());
} else if (ACTION_HELP.equals(action) ||
ACTION_USAGE.equals(action)) {
log.info(serviceArgs.usage());
@@ -1928,100 +1925,6 @@ public class SliderClient extends
AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /*
- * Creates a site conf with entries from clientProperties of ClusterStatus
- * @param desc ClusterDescription, can be null
- * @param clustername, can be null
- * @return site conf
- */
- public Configuration getSiteConf(ClusterDescription desc, String clustername)
- throws YarnException, IOException {
-if (desc == null) {
- desc = getClusterDescription();
-}
-if (clustername == null) {
- clustername = getDeployedClusterName();
-}
-String description = "Slider Application Instance " + clustername;
-
-Configuration siteConf = new Configuration(false);
-for (String key : desc.clientProperties.keySet()) {
- siteConf.set(key, desc.clientProperties.get(key), description);
-}
-return siteConf;
- }
-
-
- /**
- * get the cluster configuration
- * @param clustername cluster name
- * @return the cluster name
- */
-
- @SuppressWarnings(
-{"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs)
throws
- YarnException,
- IOException {
-File outfile = null;
-
-if (confArgs.getOutput() != null) {
- outfile = new File(confArgs.getOutput());
-}
-
-String format = confArgs.getFormat();
-verifyBindingsDefined();
-SliderUtils.validateClusterName(clustername);
-ClusterDescription status = getClusterDescription(clustername);
-Writer writer;
-boolean toPrint;
-if (outfile != null) {
- writer = new FileWriter(outfile);
- toPrint = false;
-} else {
- writer = new StringWriter();
- toPrint = true;
-}
-try {
- String description = "Slider Application Instance " + clustername;
-// JDK7
-/*
- swit
[1/7] git commit: SLIDER-493 configurable hadoop directories for accumulo app package
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 032dacf4c ->
3a759d1ec
SLIDER-493 configurable hadoop directories for accumulo app package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7d2aeb0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7d2aeb0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7d2aeb0d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7d2aeb0de92258bbc4a5ab1c16ef647df3aa6bc9
Parents: 0419a5a
Author: Billie Rinaldi
Authored: Tue Oct 7 18:26:58 2014 -0700
Committer: Billie Rinaldi
Committed: Tue Oct 7 18:26:58 2014 -0700
--
app-packages/accumulo/appConfig-default.json | 6 +++---
app-packages/accumulo/pom.xml| 7 +--
2 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d2aeb0d/app-packages/accumulo/appConfig-default.json
--
diff --git a/app-packages/accumulo/appConfig-default.json
b/app-packages/accumulo/appConfig-default.json
index 9e04564..b33261c 100644
--- a/app-packages/accumulo/appConfig-default.json
+++ b/app-packages/accumulo/appConfig-default.json
@@ -14,9 +14,9 @@
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
"site.global.other_heapsize": "128m",
-"site.global.hadoop_prefix": "${hadoop.dir}/hadoop",
+"site.global.hadoop_prefix": "${hadoop.dir}",
"site.global.hadoop_conf_dir": "/etc/hadoop/conf",
-"site.global.zookeeper_home": "${zk.dir}/zookeeper",
+"site.global.zookeeper_home": "${zk.dir}",
"site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
"site.global.accumulo_root_password": "NOT_USED",
"site.global.user_group": "${app.user.group}",
@@ -44,7 +44,7 @@
"site.accumulo-site.monitor.port.log4j": "0",
"site.accumulo-site.master.replication.coordinator.port": "0",
"site.accumulo-site.replication.receipt.service.port": "0",
-"site.accumulo-site.general.classpaths":
"$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/hadoop/.*.jar,\n${hadoop.dir}/hadoop/lib/.*.jar,\n${hadoop.dir}/hadoop-hdfs/.*.jar,\n${hadoop.dir}/hadoop-mapreduce/.*.jar,\n${hadoop.dir}/hadoop-yarn/.*.jar,"
+"site.accumulo-site.general.classpaths":
"$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/.*.jar,\n${hadoop.dir}/lib/.*.jar,\n${hdfs.dir}/.*.jar,\n${mapred.dir}/.*.jar,\n${yarn.dir}/.*.jar,"
},
"credentials": {
"jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks":
["root.initial.password", "instance.secret", "trace.token.property.password"]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7d2aeb0d/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 4cd5cca..5dae559 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -39,8 +39,11 @@
${project.build.directory}
${project.build.directory}/test-config
-/usr/lib
-${hadoop.dir}
+/usr/lib/hadoop
+/usr/lib/hadoop-hdfs
+/usr/lib/hadoop-yarn
+/usr/lib/hadoop-mapred
+/usr/lib/zookeeper
${java.home}
yarn
hadoop
[2/7] git commit: SLIDER-474 attempt fix for accumulo funtests
SLIDER-474 attempt fix for accumulo funtests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6b19cb81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6b19cb81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6b19cb81
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6b19cb81b564a8e6f8b57de5f30855e2200d30da
Parents: 7d2aeb0
Author: Billie Rinaldi
Authored: Wed Oct 8 07:20:18 2014 -0700
Committer: Billie Rinaldi
Committed: Wed Oct 8 07:20:18 2014 -0700
--
app-packages/accumulo/appConfig-default.json | 3 ++-
app-packages/accumulo/pom.xml| 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6b19cb81/app-packages/accumulo/appConfig-default.json
--
diff --git a/app-packages/accumulo/appConfig-default.json
b/app-packages/accumulo/appConfig-default.json
index b33261c..de370bf 100644
--- a/app-packages/accumulo/appConfig-default.json
+++ b/app-packages/accumulo/appConfig-default.json
@@ -51,7 +51,8 @@
},
"components": {
"slider-appmaster": {
- "jvm.heapsize": "256M"
+ "jvm.heapsize": "256M",
+ "slider.am.keytab.local.path": "${accumulo.headless.keytab}"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6b19cb81/app-packages/accumulo/pom.xml
--
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 5dae559..dc804ae 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -49,6 +49,7 @@
hadoop
+${accumulo.keytab}
[3/7] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9175aa7e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9175aa7e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9175aa7e Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 9175aa7e5acd704faf9075076f5b1be61b3bb70f Parents: 032dacf 6b19cb8 Author: Steve Loughran Authored: Wed Oct 8 09:55:26 2014 -0700 Committer: Steve Loughran Committed: Wed Oct 8 09:55:26 2014 -0700 -- app-packages/accumulo/appConfig-default.json | 9 + app-packages/accumulo/pom.xml| 8 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) --
[6/7] git commit: Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry (patch SLIDER-408)
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
(patch SLIDER-408)
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02623b1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02623b1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02623b1d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 02623b1d73058cfb081098f712d699a1d57bc519
Parents: cc2a975 e375726
Author: Steve Loughran
Authored: Wed Oct 8 11:45:43 2014 -0700
Committer: Steve Loughran
Committed: Wed Oct 8 11:45:43 2014 -0700
--
.../org/apache/slider/client/SliderClient.java | 97
.../slider/common/params/ActionGetConfArgs.java | 51 --
.../apache/slider/common/params/ClientArgs.java | 9 --
.../slider/common/params/SliderActions.java | 1 -
.../slider/client/TestCommonArgParsing.groovy | 31 ---
.../funtest/framework/CommandTestBase.groovy| 13 ---
.../commands/UnknownClusterOperationsIT.groovy | 6 --
.../lifecycle/AgentClusterLifecycleIT.groovy| 2 -
.../funtest/HBaseClusterLifecycleIT.groovy | 5 +-
9 files changed, 4 insertions(+), 211 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02623b1d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index c5022ee,8d2bcfd..6613117
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -70,10 -58,8 +70,9 @@@ import org.apache.slider.common.params.
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
- import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
@@@ -339,15 -319,11 +338,13 @@@ public class SliderClient extends Abstr
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
+} else if (ACTION_DIAGNOSTIC.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
- } else if (ACTION_GETCONF.equals(action)) {
- exitCode = actionGetConf(clusterName,
serviceArgs.getActionGetConfArgs());
} else if (ACTION_HELP.equals(action) ||
ACTION_USAGE.equals(action)) {
log.info(serviceArgs.usage());
@@@ -1891,101 -1925,10 +1888,7 @@@
return EXIT_SUCCESS;
}
- /*
-* Creates a site conf with entries from clientProperties of ClusterStatus
-* @param desc ClusterDescription, can be null
-* @param clustername, can be null
-* @return site conf
-*/
- public Configuration getSiteConf(ClusterDescription desc, String
clustername)
- throws YarnException, IOException {
- if (desc == null) {
- desc = getClusterDescription();
- }
- if (clustername == null) {
- clustername = getDeployedClusterName();
- }
- String description = "Slider Application Instance " + clustername;
-
- Configuration siteConf = new Configuration(false);
- for (String key : desc.clientProperties.keySet()) {
- siteConf.set(key, desc.clientProperties.get(key), description);
- }
- return siteConf;
- }
--
-
-- /**
-* get the cluster configuration
-* @param clustername cluster name
-* @return the cluster name
-*/
-
- @SuppressWarnings(
- {"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs)
- throws YarnException, IOException {
- File outfile = null;
-
- if (confArgs.getOutput() != null) {
- outf
