Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/733745ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/733745ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/733745ea
Branch: refs/heads/develop
Commit: 733745eaf1cf3734dd8f25e7ff0945c253c4777f
Parents: f058495 41ec741
Author: Steve Loughran <ste...@apache.org>
Authored: Tue Sep 30 17:48:02 2014 -0700
Committer: Steve Loughran <ste...@apache.org>
Committed: Tue Sep 30 17:48:02 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 57 ++
app-packages/accumulo/appConfig.json | 57 --
app-packages/accumulo/pom.xml | 10 +
app-packages/accumulo/resources-default.json | 39 ++
app-packages/accumulo/resources.json | 39 --
app-packages/accumulo/src/assembly/accumulo.xml | 4 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +-
app-packages/hbase-win/README.txt | 3 -
app-packages/hbase-win/appConfig-default.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/hbase/resources-default.json | 3 +
app-packages/storm-win/README.txt | 36 ++
app-packages/storm-win/appConfig-default.json | 41 ++
.../storm-win/configuration/storm-env.xml | 65 +++
.../storm-win/configuration/storm-site.xml | 580 +++++++++++++++++++
app-packages/storm-win/metainfo.xml | 149 +++++
.../storm-win/package/scripts/drpc_server.py | 55 ++
.../storm-win/package/scripts/nimbus.py | 55 ++
.../storm-win/package/scripts/params.py | 56 ++
.../storm-win/package/scripts/rest_api.py | 57 ++
.../storm-win/package/scripts/service.py | 56 ++
.../storm-win/package/scripts/status_params.py | 37 ++
app-packages/storm-win/package/scripts/storm.py | 53 ++
.../storm-win/package/scripts/supervisor.py | 61 ++
.../storm-win/package/scripts/ui_server.py | 55 ++
.../storm-win/package/scripts/yaml_config.py | 80 +++
.../storm-win/package/templates/config.yaml.j2 | 28 +
.../package/templates/storm_jaas.conf.j2 | 44 ++
app-packages/storm-win/pom.xml | 91 +++
app-packages/storm-win/resources-default.json | 30 +
app-packages/storm-win/src/assembly/storm.xml | 68 +++
app-packages/storm/appConfig-default.json | 2 +-
slider-agent/src/main/python/kazoo/client.py | 56 +-
.../src/main/python/kazoo/handlers/utils.py | 42 +-
.../main/python/kazoo/protocol/connection.py | 74 +--
.../src/main/python/kazoo/tests/test_client.py | 55 +-
.../main/python/kazoo/tests/test_connection.py | 89 ++-
slider-core/pom.xml | 6 +
.../org/apache/slider/client/SliderClient.java | 291 +++++++++-
.../common/SliderXMLConfKeysForTesting.java | 1 +
.../common/params/ActionDiagnosticArgs.java | 66 +++
.../apache/slider/common/params/Arguments.java | 7 +
.../apache/slider/common/params/ClientArgs.java | 11 +-
.../slider/common/params/SliderActions.java | 4 +
.../apache/slider/common/tools/SliderUtils.java | 108 ++++
.../slider/providers/agent/AgentKeys.java | 5 +
.../providers/agent/AgentProviderService.java | 2 +-
.../services/security/CertificateManager.java | 48 +-
.../providers/slideram/instance/appconf.json | 3 +-
.../agent/TestAgentAMManagementWS.groovy | 131 +++--
.../apache/slider/test/SliderTestUtils.groovy | 3 +-
.../framework/AgentCommandTestBase.groovy | 7 +
.../funtest/lifecycle/AMFailuresIT.groovy | 6 +
.../clusters/remote/slider/slider-client.xml | 6 +
54 files changed, 2595 insertions(+), 344 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 99896c8,06c37ba..5d994f1
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -133,8 -136,8 +142,9 @@@ import java.io.IOException
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
+ import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@@ -2408,55 -2377,297 +2428,318 @@@ public class SliderClient extends Abstr
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public List<ServiceInstanceData> actionRegistryList(
+ public Collection<ServiceRecord> actionRegistryListYarn(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- SliderRegistryService registryService = getRegistry();
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- registryService.findInstances(serviceType, name);
- int size = instances.size();
- if (size == 0) {
- throw new FileNotFoundException("No entries for servicetype "
- + serviceType
- + " name " + name);
+ RegistryOperations operations = getRegistryOperations();
+ Collection<ServiceRecord> serviceRecords;
+ if (StringUtils.isEmpty(name)) {
+ String serviceclassPath =
+ serviceclassPath(
+ currentUser(),
+ serviceType);
+
+ try {
+ Map<String, ServiceRecord> recordMap =
+ listServiceRecords(operations, serviceclassPath);
+ RegistryPathStatus[] listDir;
+ if (recordMap.isEmpty()) {
+ throw new UnknownApplicationInstanceException(
+ "No applications registered under " + serviceclassPath);
+ }
+ serviceRecords = recordMap.values();
+ } catch (PathNotFoundException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(),
+ e);
+ }
+ } else {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+ serviceRecords = new ArrayList<ServiceRecord>(1);
+ serviceRecords.add(instance);
}
- List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
- for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
- ServiceInstanceData payload = instance.payload;
- logInstance(payload, registryArgs.verbose);
- sids.add(payload);
+
+ for (ServiceRecord serviceRecord : serviceRecords) {
+ logInstance(serviceRecord, registryArgs.verbose);
}
- return sids;
+ return serviceRecords;
}
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException
+ * YARN problems
+ * @throws IOException
+ * Network or other problems
+ */
+ private int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ try {
+ if (diagnosticArgs.client) {
+ actionDiagnosticClient();
+ } else if
(SliderUtils.isSet(diagnosticArgs.application)) {
+ actionDiagnosticApplication(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.slider)) {
+ actionDiagnosticSlider(diagnosticArgs);
+ } else if (diagnosticArgs.yarn) {
+ actionDiagnosticYarn(diagnosticArgs);
+ } else if (diagnosticArgs.credentials) {
+ actionDiagnosticCredentials();
+ } else if (SliderUtils.isSet(diagnosticArgs.all)) {
+ actionDiagnosticAll(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.level)) {
+ actionDiagnosticIntelligent(diagnosticArgs);
+ } else {
+ // it's an unknown command
+ log.info(ActionDiagnosticArgs.USAGE);
+ return EXIT_USAGE;
+ }
+ } catch (Exception e) {
+ log.error(e.toString());
+ return EXIT_FALSE;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ private void actionDiagnosticIntelligent(ActionDiagnosticArgs
diagnosticArgs)
+ throws YarnException, IOException, URISyntaxException {
+ // not using member variable clustername because we want to
place
+ // application name after --application option and member
variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.level;
+
+ try {
+ SliderUtils.validateClientConfigFile();
+ log.info("Slider-client.xml is accessible");
+ } catch (IOException e) {
+ // we are catching exceptions here because those are
indication of
+ // validation result, and we need to print them here
+ log.error("validation of slider-client.xml fails
because: "
+ + e.toString());
+ return;
+ }
+ SliderClusterOperations clusterOperations =
createClusterOperations(clusterName);
+ // cluster not found exceptions will be thrown upstream
+ ClusterDescription clusterDescription = clusterOperations
+ .getClusterDescription();
+ log.info("Slider AppMaster is accessible");
+
+ if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
+ AggregateConf instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ String imagePath =
instanceDefinition.getInternalOperations().get(
+
InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, that means slider uploaded the agent tarball
for the user
+ //and we need to use where slider has put
+ if(imagePath == null){
- ApplicationReport appReport =
YARNRegistryClient.findInstance(clusterName);
++ ApplicationReport appReport =
findInstance(clusterName);
+ Path path1 =
sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1,
appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem,
imagePath);
+ log.info("Slider agent tarball is properly
installed");
+ } catch (IOException e) {
+ log.error("can not find or open agent tar ball:
" + e.toString());
+ return;
+ }
+ String pkgTarballPath =
instanceDefinition.getAppConfOperations()
+
.getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem,
pkgTarballPath);
+ log.info("Application tarball is properly
installed");
+ } catch (IOException e) {
+ log.error("can not find or open application tar
ball: "
+ + e.toString());
+ return;
+ }
+ }
+ }
+
+ private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs)
+ throws IOException, YarnException {
+ //assign application name from param to each sub diagnostic
function
+ diagnosticArgs.application = diagnosticArgs.all;
+ diagnosticArgs.slider = diagnosticArgs.all;
+ actionDiagnosticClient();
+ actionDiagnosticApplication(diagnosticArgs);
+ actionDiagnosticSlider(diagnosticArgs);
+ actionDiagnosticYarn(diagnosticArgs);
+ actionDiagnosticCredentials();
+ }
+
+ private void actionDiagnosticCredentials() throws BadConfigException,
IOException
+ {
+ if (SliderUtils.isHadoopClusterSecure(SliderUtils
+ .loadClientConfigurationResource())) {
+ String credentialCacheFileDescription = null;
+ try {
+ credentialCacheFileDescription = SliderUtils
+ .checkCredentialCacheFile();
+ } catch (BadConfigException e) {
+ log.error("The credential config is not valid:
" + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Unable to read the credential file:
" + e.toString());
+ throw e;
+ }
+ log.info("Credential cache file for the current user: "
+ + credentialCacheFileDescription);
+ } else {
+ log.info("the cluster is not in secure mode");
+ }
+ }
+
+ private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs)
throws IOException, YarnException {
+ JSONObject converter = null;
+ log.info("the node in the YARN cluster has below state: ");
+ List<NodeReport> yarnClusterInfo;
+ try {
+ yarnClusterInfo =
yarnClient.getNodeReports(NodeState.RUNNING);
+ } catch (YarnException e1) {
+ log.error("Exception happened when fetching node report
from the YARN cluster: " + e1.toString());
+ throw e1;
+ } catch (IOException e1) {
+ log.error("Network problem happened when fetching node
report YARN cluster: " + e1.toString());
+ throw e1;
+ }
+ for(NodeReport nodeReport : yarnClusterInfo){
+ log.info(nodeReport.toString());
+ }
+
+ if (diagnosticArgs.verbose) {
+ Writer configWriter = new StringWriter();
+ try {
+
Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter);
+ } catch (IOException e1) {
+ log.error("Network problem happened when
retrieving YARN config from YARN: " + e1.toString());
+ throw e1;
+ }
+ try {
+ converter = new
JSONObject(configWriter.toString());
+ log.info("the configuration of the YARN cluster
is: "
+ + converter.toString(2));
+
+ } catch (JSONException e) {
+ log.error("JSONException happened during
parsing response from YARN: " + e.toString());
+ }
+ }
+ }
+
+ private void actionDiagnosticSlider(ActionDiagnosticArgs
diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to
place
+ // application name after --application option and member
variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.slider;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations =
createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance
definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving
instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String imagePath =
instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, it will be uploaded by Slider and thus at slider's
path
+ if(imagePath == null){
- ApplicationReport appReport =
YARNRegistryClient.findInstance(clusterName);
++ ApplicationReport appReport = findInstance(clusterName);
+ Path path1 =
sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1,
appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ log.info("The path of slider agent tarball on HDFS is: " +
imagePath);
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs
diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to
place
+ // application name after --application option and member
variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.application;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations =
createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance
definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving
instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String clusterDir = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_ROOT);
+ String pkgTarball = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_DEF);
+ String runAsUser = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.RUNAS_USER);
+
+ log.info("The location of the cluster instance directory in
HDFS is: "
+ + clusterDir);
+ log.info("The name of the application package tarball on HDFS
is: "
+ + pkgTarball);
+ log.info("The runas user of the application in the cluster is: "
+ + runAsUser);
+
+ if (diagnosticArgs.verbose) {
+ log.info("App config of the application: "
+ +
instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application: "
+ +
instanceDefinition.getResources().toJson());
+ }
+ }
+
+ private void actionDiagnosticClient() throws SliderException,
IOException {
+ String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+ String clientConfigPath = SliderUtils.getClientConfigPath();
+ String jdkInfo = SliderUtils.getJDKInfo();
+ log.info("The slider command path: " + currentCommandPath);
+ log.info("The slider-client.xml used by current running command
path: "
+ + clientConfigPath);
+ log.info(jdkInfo);
+
+ try {
+ SliderUtils.validateSliderClientEnvironment(log);
+ } catch (SliderException e) {
+ log.error(e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error(e.toString());
+ throw e;
+ }
+ }
+
- private void logInstance(ServiceInstanceData instance,
++
+ /**
+ * Log a service record instance
+ * @param instance record
+ * @param verbose verbose logging of all external endpoints
+ */
+ private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.id);
+ log.info("{}", instance.yarn_id);
} else {
- log.info("{}: ", instance.id);
+ log.info("{}: ", instance.yarn_id);
logEndpoints(instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
