Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e883bf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e883bf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e883bf9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6e883bf91de781fa8ec5ca7bb1f98f2b9fd8a0dd
Parents: c5fb4f0 82cf1f0
Author: Steve Loughran <ste...@apache.org>
Authored: Mon Sep 1 16:55:59 2014 +0100
Committer: Steve Loughran <ste...@apache.org>
Committed: Mon Sep 1 16:55:59 2014 +0100
----------------------------------------------------------------------
README.md | 4 +-
app-packages/hbase/appConfig.json | 74 +++++-----
app-packages/hbase/pom.xml | 2 +-
.../src/main/python/agent/Controller.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 10 +-
slider-agent/src/main/python/agent/Register.py | 5 +-
.../src/test/python/agent/TestRegistration.py | 5 +-
slider-assembly/src/main/bash/README.md | 2 +-
slider-assembly/src/main/bash/slider_destroy | 4 +-
slider-assembly/src/main/scripts/slider.py | 6 +-
.../org/apache/slider/client/SliderClient.java | 10 +-
.../slider/common/params/ActionFreezeArgs.java | 2 +-
.../common/params/LaunchArgsAccessor.java | 2 +-
.../slider/common/params/SliderActions.java | 8 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 147 ++++++++++++-------
.../server/appmaster/state/NodeEntry.java | 2 +-
.../server/appmaster/state/RoleHistory.java | 6 +-
.../appmaster/web/rest/agent/Register.java | 13 ++
.../agent/actions/TestActionExists.groovy | 2 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 10 +-
.../TestFreezeThawMasterlessAM.groovy | 8 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 2 +-
.../standalone/TestStandaloneAMDestroy.groovy | 6 +-
.../agent/standalone/TestYarnRegistryAM.groovy | 2 +-
.../slider/client/TestCommonArgParsing.groovy | 2 +-
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 13 +-
.../agent/TestAgentProviderService.java | 35 ++++-
.../src/test/resources/example-slider-test.xml | 4 +-
.../funtest/framework/CommandTestBase.groovy | 2 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 8 +-
.../src/test/manual/python/SliderTester.py | 4 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../funtest/HBaseClusterLifecycleIT.groovy | 8 +-
.../TestFreezeThawClusterFromArchive.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
.../TestFreezeThawLiveRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 4 +-
src/test/clusters/sandbox/operations.md | 20 +--
src/test/clusters/ubuntu-secure/operations.md | 14 +-
43 files changed, 286 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --cc
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a91a817,61866fb..4bfc718
---
a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++
b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -582,11 -589,11 +592,11 @@@ public class AgentProviderService exten
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn,
roleName);
-
+
// and update registration entries
if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
- TimeUnit.MILLISECONDS));
+ queueAccess.put(new RegisterComponentInstance(instance.getId(),
+ roleName, 0, TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --cc
slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
index bdb5b27,0000000..bd15bb4
mode 100644,000000..100644
---
a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++
b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
@@@ -1,379 -1,0 +1,379 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
+
+import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
+import org.apache.slider.core.registry.docstore.UriMap
+import org.apache.slider.core.registry.info.CustomRegistryConstants
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Test
+
+/**
+ * work with a YARN registry
+ */
+@CompileStatic
+@Slf4j
+
+class TestYarnRegistryAM extends AgentMiniClusterTestBase {
+
+
+ public static final String ARTIFACT_NAME =
PublishedArtifacts.COMPLETE_CONFIG
+
+ @Test
+ public void testYarnRegistryAM() throws Throwable {
+
+
+ describe "create a masterless AM then perform YARN registry operations on
it"
+
+
+ String clustername = createMiniCluster(configuration, 1, true)
+
+ // get local binding
+ def registryOperations = microZKCluster.registryOperations
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+
+ // verify the cluster has the YARN reg service live
+ def rmRegistryService =
miniCluster.getResourceManager(0).getRMContext().registry
+ assert rmRegistryService
+
+
+
+
+
+ ServiceLauncher<SliderClient> launcher
+ launcher = createStandaloneAM(clustername, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)clusterNodes).size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def yarnRegistryClient = client.yarnAppListClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ // sleep to allow registration to complete
+ sleep(5000)
+
+
+
+
+ try {
+ def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ log.info("yarn service registry: \n${yarnRegistryDump}\n")
+ } catch (IOException ignored) {
+
+ }
+
+
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ def self = currentUser()
+ RegistryPathStatus[] serviceTypes =
registryService.listDir(userPath(self))
+ dumpArray(serviceTypes)
+
+ def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
+
+ def serviceRecords =
RecordOperations.extractServiceRecords(registryService,
+ registryService.listDir(recordsPath))
+ dumpCollection(serviceRecords)
+ assert serviceRecords.size() == 1
+
+ def serviceInstance = serviceRecords[0]
+ log.info(serviceInstance.toString())
+
+ assert 2 <= serviceInstance.external.size()
+
+ // hit the registry web page
+
+ def registryEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL =
RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+ describe("Registry WADL @ $registryURL")
+
+ def publisherEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API)
+
+ def publisherURL =
RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
+ def setlisting = GET(publisherURL)
+
+ log.info(setlisting)
+
+ UriMap uris = uriMapDeser.fromJson(setlisting)
+ assert uris.uris[RestPaths.SLIDER_CONFIGSET]
+ def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
+
+ assert publishedYarnSite.empty
+
+ //get the full URL
+ def yarnSitePublisher = appendToURL(publisher,
+ RestPaths.SLIDER_CONFIGSET,
+ ARTIFACT_NAME)
+
+ String confJSON = GET(yarnSitePublisher)
+// log.info(confJSON)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+
+ publishedYarnSite = confSerDeser.fromJson(confJSON)
+
+ assert !publishedYarnSite.empty
+
+
+ //get the XML
+ def yarnSiteXML = yarnSitePublisher + ".xml"
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String properties = GET(yarnSitePublisher + ".properties")
+ Properties parsedProps = new Properties()
+ parsedProps.load(new StringReader(properties))
+ assert parsedProps.size() > 0
+ def rmAddrFromDownloadedProperties =
parsedProps.get(YarnConfiguration.RM_ADDRESS)
+ def rmHostnameFromDownloadedProperties =
parsedProps.get(YarnConfiguration.RM_HOSTNAME)
+ assert rmAddrFromDownloadedProperties
+ assert rmHostnameFromDownloadedProperties
+
+ String json = GET(yarnSitePublisher + ".json")
+
+
+
+ describe("Registry List")
+ log.info(GET(registryURL))
+
+
+ describe "Registry Retrieval Class"
+ // retrieval
+
+ RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[ARTIFACT_NAME]
+
+
+ describe "verify SLIDER-52 processing"
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ ARTIFACT_NAME,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
+ def rmHostnameViaClientSideXML = parsedProps.get(
+ YarnConfiguration.RM_HOSTNAME)
+ assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
+ def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
+ describe "fetch missing artifact"
+ try {
+ retriever.retrieveConfiguration(externalConfSet, "no-such-artifact",
true)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ try {
+ retriever.getConfigurations(false)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.verbose = true
+
+ // list all
+ registryArgs.list = true;
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+
+ // list a named instance and expect a failure
+ registryArgs.list = true;
+ registryArgs.name = "unknown"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ // list all instances of an alternate type and expect failure
+ registryArgs.list = true;
+ registryArgs.name = null
+ registryArgs.serviceType = "org-apache-hadoop"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ registryArgs.serviceType = ""
+
+ //set the name
+ registryArgs.name = clustername;
+ registryArgs.serviceType = SliderKeys.APP_TYPE
+
+
+ //now expect list to work
+ describe registryArgs.toString()
+
+ def listedInstance = client.actionRegistryListYarn(registryArgs)
+ assert listedInstance[0].id == serviceInstance.id
+
+
+ // listconf
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+ // listconf --internal
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ registryArgs.internal = true
+ describe registryArgs.toString()
+ assert SliderExitCodes.EXIT_NOT_FOUND ==
client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = false
+ registryArgs.internal = false
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registryArgs.getConf = yarn_site_config
+
+ //properties format
+ registryArgs.format = "properties"
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+
+ File outputDir = new File("target/test_standalone_registry_am/output")
+ outputDir.mkdirs()
+
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(clustername)
+ registryArgs.getConf = yarn_site_config
+ registryArgs.dest = outputDir
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".xml").exists()
+
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
+ describe registryArgs.toString()
+
+ def unknownFilename = "undefined-file"
+ registryArgs.getConf = unknownFilename
+ assert SliderExitCodes.EXIT_NOT_FOUND ==
client.actionRegistry(registryArgs)
+
- describe "freeze cluster"
++ describe "stop cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = yarnRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
----------------------------------------------------------------------
