Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts: slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e883bf9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e883bf9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e883bf9 Branch: refs/heads/develop Commit: 6e883bf91de781fa8ec5ca7bb1f98f2b9fd8a0dd Parents: c5fb4f0 82cf1f0 Author: Steve Loughran <ste...@apache.org> Authored: Mon Sep 1 16:55:59 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Sep 1 16:55:59 2014 +0100 ---------------------------------------------------------------------- README.md | 4 +- app-packages/hbase/appConfig.json | 74 +++++----- app-packages/hbase/pom.xml | 2 +- .../src/main/python/agent/Controller.py | 1 + .../python/agent/CustomServiceOrchestrator.py | 10 +- slider-agent/src/main/python/agent/Register.py | 5 +- .../src/test/python/agent/TestRegistration.py | 5 +- slider-assembly/src/main/bash/README.md | 2 +- slider-assembly/src/main/bash/slider_destroy | 4 +- slider-assembly/src/main/scripts/slider.py | 6 +- .../org/apache/slider/client/SliderClient.java | 10 +- .../slider/common/params/ActionFreezeArgs.java | 2 +- .../common/params/LaunchArgsAccessor.java | 2 +- .../slider/common/params/SliderActions.java | 8 +- .../slider/providers/agent/AgentKeys.java | 1 + .../providers/agent/AgentProviderService.java | 147 ++++++++++++------- .../server/appmaster/state/NodeEntry.java | 2 +- .../server/appmaster/state/RoleHistory.java | 6 +- .../appmaster/web/rest/agent/Register.java | 13 ++ .../agent/actions/TestActionExists.groovy | 2 +- .../agent/freezethaw/TestFreezeCommands.groovy | 10 +- .../TestFreezeThawMasterlessAM.groovy | 8 +- .../freezethaw/TestFreezeUnknownCluster.groovy | 2 +- .../standalone/TestBuildStandaloneAM.groovy | 2 +- .../standalone/TestStandaloneAMDestroy.groovy | 6 +- .../agent/standalone/TestYarnRegistryAM.groovy | 2 +- .../slider/client/TestCommonArgParsing.groovy | 2 +- .../model/history/TestRoleHistoryRW.groovy | 4 +- .../slider/test/YarnMiniClusterTestBase.groovy | 13 +- .../agent/TestAgentProviderService.java | 35 ++++- .../src/test/resources/example-slider-test.xml | 4 +- .../funtest/framework/CommandTestBase.groovy | 2 +- .../lifecycle/AgentClusterLifecycleIT.groovy | 8 +- .../src/test/manual/python/SliderTester.py | 4 +- .../accumulo/live/TestAccFreezeThaw.groovy | 4 +- .../funtest/HBaseClusterLifecycleIT.groovy | 8 +- .../TestFreezeThawClusterFromArchive.groovy | 2 +- .../build/TestBuildThawClusterM1W1.groovy | 2 +- ...reezeReconfigureThawLiveRegionService.groovy | 4 +- .../TestFreezeThawLiveRegionService.groovy | 4 +- .../minicluster/live/TestTwoLiveClusters.groovy | 4 +- src/test/clusters/sandbox/operations.md | 20 +-- src/test/clusters/ubuntu-secure/operations.md | 14 +- 43 files changed, 286 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index a91a817,61866fb..4bfc718 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@@ -582,11 -589,11 +592,11 @@@ public class AgentProviderService exten // component specific publishes processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName); - + // and update registration entries if (instance != null) { - queueAccess.put(new RegisterComponentInstance(instance.getId(), 0, - TimeUnit.MILLISECONDS)); + queueAccess.put(new RegisterComponentInstance(instance.getId(), + roleName, 0, TimeUnit.MILLISECONDS)); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy index bdb5b27,0000000..bd15bb4 mode 100644,000000..100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy @@@ -1,379 -1,0 +1,379 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.agent.standalone + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.PathNotFoundException +import org.apache.hadoop.yarn.api.records.ApplicationReport +import org.apache.hadoop.yarn.api.records.YarnApplicationState +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.hadoop.yarn.registry.client.api.RegistryConstants +import org.apache.hadoop.yarn.registry.client.binding.RecordOperations +import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils +import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus + +import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.* +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.api.ClusterNode +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.ActionRegistryArgs +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.persist.JsonSerDeser +import org.apache.slider.core.registry.docstore.PublishedConfigSet +import org.apache.slider.core.registry.docstore.PublishedConfiguration +import org.apache.slider.core.registry.docstore.UriMap +import org.apache.slider.core.registry.info.CustomRegistryConstants +import org.apache.slider.core.registry.retrieve.RegistryRetriever +import org.apache.slider.server.appmaster.PublishedArtifacts +import org.apache.slider.server.appmaster.web.rest.RestPaths +import org.junit.Test + +/** + * work with a YARN registry + */ +@CompileStatic +@Slf4j + +class TestYarnRegistryAM extends AgentMiniClusterTestBase { + + + public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG + + @Test + public void testYarnRegistryAM() throws Throwable { + + + describe "create a masterless AM then perform YARN registry operations on it" + + + String clustername = createMiniCluster(configuration, 1, true) + + // get local binding + def registryOperations = microZKCluster.registryOperations + registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES) + + // verify the cluster has the YARN reg service live + def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry + assert rmRegistryService + + + + + + ServiceLauncher<SliderClient> launcher + launcher = createStandaloneAM(clustername, true, false) + SliderClient client = launcher.service + addToTeardown(client); + + ApplicationReport report = waitForClusterLive(client) + logReport(report) + List<ApplicationReport> apps = client.applications; + + List<ClusterNode> clusterNodes = client.listClusterNodesInRole( + SliderKeys.COMPONENT_AM) + assert ((List<ClusterNode>)clusterNodes).size() == 1 + + ClusterNode masterNode = clusterNodes[0] + log.info("Master node = ${masterNode}"); + + List<ClusterNode> nodes + String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM) + assert uuids.length == 1; + nodes = client.listClusterNodes(uuids); + assert ((List<ClusterNode>)nodes).size() == 1; + describe "AM Node UUID=${uuids[0]}" + + nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM) + assert ((List<ClusterNode>)nodes).size() == 1; + nodes = listNodesInRole(client, "") + assert ((List<ClusterNode>)nodes).size() == 1; + ClusterNode master = nodes[0] + assert master.role == SliderKeys.COMPONENT_AM + + + + + String username = client.username + def yarnRegistryClient = client.yarnAppListClient + describe("list of all applications") + logApplications(apps) + describe("apps of user $username") + List<ApplicationReport> userInstances = yarnRegistryClient.listInstances() + logApplications(userInstances) + assert userInstances.size() == 1 + describe("named app $clustername") + ApplicationReport instance = yarnRegistryClient.findInstance(clustername) + logReport(instance) + assert instance != null + + // sleep to allow registration to complete + sleep(5000) + + + + + try { + def yarnRegistryDump = client.dumpYarnRegistry(true).toString() + log.info("yarn service registry: \n${yarnRegistryDump}\n") + } catch (IOException ignored) { + + } + + + describe "service registry names" + def registryService = client.registryOperations + + def self = currentUser() + RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self)) + dumpArray(serviceTypes) + + def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE) + + def serviceRecords = RecordOperations.extractServiceRecords(registryService, + registryService.listDir(recordsPath)) + dumpCollection(serviceRecords) + assert serviceRecords.size() == 1 + + def serviceInstance = serviceRecords[0] + log.info(serviceInstance.toString()) + + assert 2 <= serviceInstance.external.size() + + // hit the registry web page + + def registryEndpoint = serviceInstance.getExternalEndpoint( + CustomRegistryConstants.REGISTRY_REST_API) + assert registryEndpoint != null + def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0] + describe("Registry WADL @ $registryURL") + + def publisherEndpoint = serviceInstance.getExternalEndpoint( + CustomRegistryConstants.PUBLISHER_REST_API) + + def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0] + def publisher = publisherURL.toString() + describe("Publisher") + + JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap) + def setlisting = GET(publisherURL) + + log.info(setlisting) + + UriMap uris = uriMapDeser.fromJson(setlisting) + assert uris.uris[RestPaths.SLIDER_CONFIGSET] + def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET) + JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>( + PublishedConfigSet) + def configSet = serDeser.fromJson(publishedJSON) + assert configSet.size() >= 1 + assert configSet.contains(ARTIFACT_NAME) + PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME) + + assert publishedYarnSite.empty + + //get the full URL + def yarnSitePublisher = appendToURL(publisher, + RestPaths.SLIDER_CONFIGSET, + ARTIFACT_NAME) + + String confJSON = GET(yarnSitePublisher) +// log.info(confJSON) + JsonSerDeser< PublishedConfiguration> confSerDeser = + new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration) + + publishedYarnSite = confSerDeser.fromJson(confJSON) + + assert !publishedYarnSite.empty + + + //get the XML + def yarnSiteXML = yarnSitePublisher + ".xml" + + + String confXML = GET(yarnSiteXML) + log.info("Conf XML at $yarnSiteXML = \n $confXML") + + String properties = GET(yarnSitePublisher + ".properties") + Properties parsedProps = new Properties() + parsedProps.load(new StringReader(properties)) + assert parsedProps.size() > 0 + def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS) + def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME) + assert rmAddrFromDownloadedProperties + assert rmHostnameFromDownloadedProperties + + String json = GET(yarnSitePublisher + ".json") + + + + describe("Registry List") + log.info(GET(registryURL)) + + + describe "Registry Retrieval Class" + // retrieval + + RegistryRetriever retriever = new RegistryRetriever(serviceInstance) + log.info retriever.toString() + + assert retriever.hasConfigurations(true) + PublishedConfigSet externalConfSet = retriever.getConfigurations(true) + dumpConfigurationSet(externalConfSet) + assert externalConfSet[ARTIFACT_NAME] + + + describe "verify SLIDER-52 processing" + def yarnSite = retriever.retrieveConfiguration( + externalConfSet, + ARTIFACT_NAME, + true) + assert !yarnSite.empty + def siteXML = yarnSite.asConfiguration() + def rmHostnameViaClientSideXML = parsedProps.get( + YarnConfiguration.RM_HOSTNAME) + assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties + def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS) + + log.info("RM from downloaded props = $rmAddrFromDownloadedProperties") + assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties + + describe "fetch missing artifact" + try { + retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true) + fail("expected a failure") + } catch (FileNotFoundException expected) { + // expected + } + describe "Internal configurations" + assert !retriever.hasConfigurations(false) + try { + retriever.getConfigurations(false) + fail("expected a failure") + } catch (FileNotFoundException expected) { + // expected + } + + + // retrieval via API + ActionRegistryArgs registryArgs = new ActionRegistryArgs() + registryArgs.verbose = true + + // list all + registryArgs.list = true; + describe registryArgs.toString() + client.actionRegistry(registryArgs) + + // list a named instance and expect a failure + registryArgs.list = true; + registryArgs.name = "unknown" + try { + client.actionRegistryListYarn(registryArgs) + } catch (PathNotFoundException expected) { + // expected + } + + // list all instances of an alternate type and expect failure + registryArgs.list = true; + registryArgs.name = null + registryArgs.serviceType = "org-apache-hadoop" + try { + client.actionRegistryListYarn(registryArgs) + } catch (PathNotFoundException expected) { + // expected + } + + registryArgs.serviceType = "" + + //set the name + registryArgs.name = clustername; + registryArgs.serviceType = SliderKeys.APP_TYPE + + + //now expect list to work + describe registryArgs.toString() + + def listedInstance = client.actionRegistryListYarn(registryArgs) + assert listedInstance[0].id == serviceInstance.id + + + // listconf + registryArgs.list = false; + registryArgs.listConf = true + describe registryArgs.toString() + + client.actionRegistry(registryArgs) + + // listconf --internal + registryArgs.list = false; + registryArgs.listConf = true + registryArgs.internal = true + describe registryArgs.toString() + assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) + + registryArgs.list = false; + registryArgs.listConf = false + registryArgs.internal = false + + def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG + registryArgs.getConf = yarn_site_config + + //properties format + registryArgs.format = "properties" + describe registryArgs.toString() + + client.actionRegistry(registryArgs) + + + File outputDir = new File("target/test_standalone_registry_am/output") + outputDir.mkdirs() + + // create a new registry args with the defaults back in + registryArgs = new ActionRegistryArgs(clustername) + registryArgs.getConf = yarn_site_config + registryArgs.dest = outputDir + describe registryArgs.toString() + client.actionRegistry(registryArgs) + assert new File(outputDir, yarn_site_config + ".xml").exists() + + registryArgs.format = "properties" + client.actionRegistry(registryArgs) + assert new File(outputDir, yarn_site_config + ".properties").exists() + + describe registryArgs.toString() + + def unknownFilename = "undefined-file" + registryArgs.getConf = unknownFilename + assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) + - describe "freeze cluster" ++ describe "stop cluster" + //now kill that cluster + assert 0 == clusterActionFreeze(client, clustername) + //list it & See if it is still there + ApplicationReport oldInstance = yarnRegistryClient.findInstance( + clustername) + assert oldInstance != null + assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED + + + + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy ----------------------------------------------------------------------