SLIDER-149 in sync with YARN-913-patch-002
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5784c4ab Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5784c4ab Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5784c4ab Branch: refs/heads/develop Commit: 5784c4ab5e2cae45625d2dfa745802c69d5f19ab Parents: 901a673 Author: Steve Loughran <ste...@apache.org> Authored: Tue Sep 9 16:12:46 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Sep 9 16:12:46 2014 +0100 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 7 +- .../server/appmaster/SliderAppMaster.java | 18 +- .../TestStandaloneYarnRegistryAM.groovy | 378 ++++++++++++++++++ .../agent/standalone/TestYarnRegistryAM.groovy | 379 ------------------- 4 files changed, 392 insertions(+), 390 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8219fd7..ed3bc0e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -136,6 +136,7 @@ import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -2376,13 +2377,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws IOException Network or other problems */ @VisibleForTesting - public List<ServiceRecord> actionRegistryListYarn( + public Collection<ServiceRecord> actionRegistryListYarn( ActionRegistryArgs registryArgs) throws YarnException, IOException { String serviceType = registryArgs.serviceType; String name = registryArgs.name; RegistryOperationsService operations = getRegistryOperations(); - List<ServiceRecord> serviceRecords; + Collection<ServiceRecord> serviceRecords; if (StringUtils.isEmpty(name)) { String serviceclassPath = BindingUtils.serviceclassPath(BindingUtils.currentUser(), @@ -2394,7 +2395,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe + serviceclassPath); } serviceRecords = - RecordOperations.extractServiceRecords(operations, listDir); + RecordOperations.extractServiceRecords(operations, listDir).values(); } else { ServiceRecord instance = lookupServiceRecord(registryArgs); serviceRecords = new ArrayList<ServiceRecord>(1); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 9615b6f..4b5e0b3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -52,11 +52,12 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService; import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies; import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.yarn.registry.server.services.ResourceManagerRegistryService; +import org.apache.hadoop.yarn.registry.server.services.RMRegistryOperationsService; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -929,6 +930,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService ServiceRecord serviceRecord = new ServiceRecord(); String serviceID = appid.toString(); serviceRecord.id = serviceID; + serviceRecord.persistence = PersistencePolicies.APPLICATION; serviceRecord.description = "Slider Application Master"; serviceRecord.addExternalEndpoint( @@ -967,7 +969,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // and an ephemeral binding to the app yarnRegistryOperations.putComponent( - RegistryTypeUtils.yarnIdToDnsId(appAttemptID.toString()), + RegistryPathUtils.encodeYarnID(appAttemptID.toString()), serviceRecord, false); @@ -982,10 +984,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ protected void setupInitialRegistryPaths() throws IOException { - if (registryOperations instanceof ResourceManagerRegistryService) { - ResourceManagerRegistryService rmRegOperations = - (ResourceManagerRegistryService) registryOperations; - rmRegOperations.createUserPath(service_user_name); + if (registryOperations instanceof RMRegistryOperationsService) { + RMRegistryOperationsService rmRegOperations = + (RMRegistryOperationsService) registryOperations; + rmRegOperations.createHomeDirectory(service_user_name); } } @@ -1003,7 +1005,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } // this is where component registrations will go log.info("Registering component {}", id); - String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString()); + String cid = RegistryPathUtils.encodeYarnID(id.toString()); ServiceRecord container = new ServiceRecord( cid, description, @@ -1027,7 +1029,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ public void unregisterComponent(ContainerId id) { log.info("Unregistering component {}", id); - String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString()); + String cid = RegistryPathUtils.encodeYarnID(id.toString()); try { yarnRegistryOperations.rmComponent(cid); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy new file mode 100644 index 0000000..c7291a8 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.agent.standalone + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.PathNotFoundException +import org.apache.hadoop.yarn.api.records.ApplicationReport +import org.apache.hadoop.yarn.api.records.YarnApplicationState +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.hadoop.yarn.registry.client.api.RegistryConstants +import org.apache.hadoop.yarn.registry.client.binding.RecordOperations +import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils +import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus +import org.apache.hadoop.yarn.registry.client.types.ServiceRecord + +import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.* +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.api.ClusterNode +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.ActionRegistryArgs +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.persist.JsonSerDeser +import org.apache.slider.core.registry.docstore.PublishedConfigSet +import org.apache.slider.core.registry.docstore.PublishedConfiguration +import org.apache.slider.core.registry.docstore.UriMap +import org.apache.slider.core.registry.info.CustomRegistryConstants +import org.apache.slider.core.registry.retrieve.RegistryRetriever +import org.apache.slider.server.appmaster.PublishedArtifacts +import org.apache.slider.server.appmaster.web.rest.RestPaths +import org.junit.Test + +/** + * work with a YARN registry + */ +@CompileStatic +@Slf4j + +class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { + + + public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG + + @Test + public void testYarnRegistryAM() throws Throwable { + + + describe "create a masterless AM then perform YARN registry operations on it" + + + String clustername = createMiniCluster(configuration, 1, true) + + // get local binding + def registryOperations = microZKCluster.registryOperations + registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES) + + // verify the cluster has the YARN reg service live + def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry + assert rmRegistryService + + ServiceLauncher<SliderClient> launcher + launcher = createStandaloneAM(clustername, true, false) + SliderClient client = launcher.service + addToTeardown(client); + + ApplicationReport report = waitForClusterLive(client) + logReport(report) + List<ApplicationReport> apps = client.applications; + + List<ClusterNode> clusterNodes = client.listClusterNodesInRole( + SliderKeys.COMPONENT_AM) + assert ((List<ClusterNode>)clusterNodes).size() == 1 + + ClusterNode masterNode = clusterNodes[0] + log.info("Master node = ${masterNode}"); + + List<ClusterNode> nodes + String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM) + assert uuids.length == 1; + nodes = client.listClusterNodes(uuids); + assert ((List<ClusterNode>)nodes).size() == 1; + describe "AM Node UUID=${uuids[0]}" + + nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM) + assert ((List<ClusterNode>)nodes).size() == 1; + nodes = listNodesInRole(client, "") + assert ((List<ClusterNode>)nodes).size() == 1; + ClusterNode master = nodes[0] + assert master.role == SliderKeys.COMPONENT_AM + + + + + String username = client.username + def yarnRegistryClient = client.yarnAppListClient + describe("list of all applications") + logApplications(apps) + describe("apps of user $username") + List<ApplicationReport> userInstances = yarnRegistryClient.listInstances() + logApplications(userInstances) + assert userInstances.size() == 1 + describe("named app $clustername") + ApplicationReport instance = yarnRegistryClient.findInstance(clustername) + logReport(instance) + assert instance != null + + // sleep to allow registration to complete + sleep(5000) + + + + + try { + def yarnRegistryDump = client.dumpYarnRegistry(true).toString() + log.info("yarn service registry: \n${yarnRegistryDump}\n") + } catch (IOException ignored) { + + } + + + describe "service registry names" + def registryService = client.registryOperations + + def self = currentUser() + RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self)) + dumpArray(serviceTypes) + + def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE) + + Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords( + registryService, + registryService.listDir(recordsPath)) + def serviceRecords = recordMap.values(); + dumpCollection(serviceRecords) + assert serviceRecords.size() == 1 + + def serviceInstance = serviceRecords[0] + log.info(serviceInstance.toString()) + + assert 2 <= serviceInstance.external.size() + + // hit the registry web page + + def registryEndpoint = serviceInstance.getExternalEndpoint( + CustomRegistryConstants.REGISTRY_REST_API) + assert registryEndpoint != null + def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0] + describe("Registry WADL @ $registryURL") + + def publisherEndpoint = serviceInstance.getExternalEndpoint( + CustomRegistryConstants.PUBLISHER_REST_API) + + def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0] + def publisher = publisherURL.toString() + describe("Publisher") + + JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap) + def setlisting = GET(publisherURL) + + log.info(setlisting) + + UriMap uris = uriMapDeser.fromJson(setlisting) + assert uris.uris[RestPaths.SLIDER_CONFIGSET] + def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET) + JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>( + PublishedConfigSet) + def configSet = serDeser.fromJson(publishedJSON) + assert configSet.size() >= 1 + assert configSet.contains(ARTIFACT_NAME) + PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME) + + assert publishedYarnSite.empty + + //get the full URL + def yarnSitePublisher = appendToURL(publisher, + RestPaths.SLIDER_CONFIGSET, + ARTIFACT_NAME) + + String confJSON = GET(yarnSitePublisher) +// log.info(confJSON) + JsonSerDeser< PublishedConfiguration> confSerDeser = + new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration) + + publishedYarnSite = confSerDeser.fromJson(confJSON) + + assert !publishedYarnSite.empty + + + //get the XML + def yarnSiteXML = yarnSitePublisher + ".xml" + + + String confXML = GET(yarnSiteXML) + log.info("Conf XML at $yarnSiteXML = \n $confXML") + + String properties = GET(yarnSitePublisher + ".properties") + Properties parsedProps = new Properties() + parsedProps.load(new StringReader(properties)) + assert parsedProps.size() > 0 + def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS) + def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME) + assert rmAddrFromDownloadedProperties + assert rmHostnameFromDownloadedProperties + + String json = GET(yarnSitePublisher + ".json") + + + + describe("Registry List") + log.info(GET(registryURL)) + + + describe "Registry Retrieval Class" + // retrieval + + RegistryRetriever retriever = new RegistryRetriever(serviceInstance) + log.info retriever.toString() + + assert retriever.hasConfigurations(true) + PublishedConfigSet externalConfSet = retriever.getConfigurations(true) + dumpConfigurationSet(externalConfSet) + assert externalConfSet[ARTIFACT_NAME] + + + describe "verify SLIDER-52 processing" + def yarnSite = retriever.retrieveConfiguration( + externalConfSet, + ARTIFACT_NAME, + true) + assert !yarnSite.empty + def siteXML = yarnSite.asConfiguration() + def rmHostnameViaClientSideXML = parsedProps.get( + YarnConfiguration.RM_HOSTNAME) + assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties + def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS) + + log.info("RM from downloaded props = $rmAddrFromDownloadedProperties") + assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties + + describe "fetch missing artifact" + try { + retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true) + fail("expected a failure") + } catch (FileNotFoundException expected) { + // expected + } + describe "Internal configurations" + assert !retriever.hasConfigurations(false) + try { + retriever.getConfigurations(false) + fail("expected a failure") + } catch (FileNotFoundException expected) { + // expected + } + + + // retrieval via API + ActionRegistryArgs registryArgs = new ActionRegistryArgs() + registryArgs.verbose = true + + // list all + registryArgs.list = true; + describe registryArgs.toString() + client.actionRegistry(registryArgs) + + // list a named instance and expect a failure + registryArgs.list = true; + registryArgs.name = "unknown" + try { + client.actionRegistryListYarn(registryArgs) + } catch (PathNotFoundException expected) { + // expected + } + + // list all instances of an alternate type and expect failure + registryArgs.list = true; + registryArgs.name = null + registryArgs.serviceType = "org-apache-hadoop" + try { + client.actionRegistryListYarn(registryArgs) + } catch (PathNotFoundException expected) { + // expected + } + + registryArgs.serviceType = "" + + //set the name + registryArgs.name = clustername; + registryArgs.serviceType = SliderKeys.APP_TYPE + + + //now expect list to work + describe registryArgs.toString() + + def listedInstance = client.actionRegistryListYarn(registryArgs) + assert listedInstance[0].id == serviceInstance.id + + + // listconf + registryArgs.list = false; + registryArgs.listConf = true + describe registryArgs.toString() + + client.actionRegistry(registryArgs) + + // listconf --internal + registryArgs.list = false; + registryArgs.listConf = true + registryArgs.internal = true + describe registryArgs.toString() + assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) + + registryArgs.list = false; + registryArgs.listConf = false + registryArgs.internal = false + + def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG + registryArgs.getConf = yarn_site_config + + //properties format + registryArgs.format = "properties" + describe registryArgs.toString() + + client.actionRegistry(registryArgs) + + + File outputDir = new File("target/test_standalone_registry_am/output") + outputDir.mkdirs() + + // create a new registry args with the defaults back in + registryArgs = new ActionRegistryArgs(clustername) + registryArgs.getConf = yarn_site_config + registryArgs.dest = outputDir + describe registryArgs.toString() + client.actionRegistry(registryArgs) + assert new File(outputDir, yarn_site_config + ".xml").exists() + + registryArgs.format = "properties" + client.actionRegistry(registryArgs) + assert new File(outputDir, yarn_site_config + ".properties").exists() + + describe registryArgs.toString() + + def unknownFilename = "undefined-file" + registryArgs.getConf = unknownFilename + assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) + + describe "stop cluster" + //now kill that cluster + assert 0 == clusterActionFreeze(client, clustername) + //list it & See if it is still there + ApplicationReport oldInstance = yarnRegistryClient.findInstance( + clustername) + assert oldInstance != null + assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED + + + + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy deleted file mode 100644 index bd15bb4..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.agent.standalone - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.PathNotFoundException -import org.apache.hadoop.yarn.api.records.ApplicationReport -import org.apache.hadoop.yarn.api.records.YarnApplicationState -import org.apache.hadoop.yarn.conf.YarnConfiguration -import org.apache.hadoop.yarn.registry.client.api.RegistryConstants -import org.apache.hadoop.yarn.registry.client.binding.RecordOperations -import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils -import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus - -import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.* -import org.apache.slider.agent.AgentMiniClusterTestBase -import org.apache.slider.api.ClusterNode -import org.apache.slider.client.SliderClient -import org.apache.slider.common.SliderExitCodes -import org.apache.slider.common.SliderKeys -import org.apache.slider.common.params.ActionRegistryArgs -import org.apache.slider.core.main.ServiceLauncher -import org.apache.slider.core.persist.JsonSerDeser -import org.apache.slider.core.registry.docstore.PublishedConfigSet -import org.apache.slider.core.registry.docstore.PublishedConfiguration -import org.apache.slider.core.registry.docstore.UriMap -import org.apache.slider.core.registry.info.CustomRegistryConstants -import org.apache.slider.core.registry.retrieve.RegistryRetriever -import org.apache.slider.server.appmaster.PublishedArtifacts -import org.apache.slider.server.appmaster.web.rest.RestPaths -import org.junit.Test - -/** - * work with a YARN registry - */ -@CompileStatic -@Slf4j - -class TestYarnRegistryAM extends AgentMiniClusterTestBase { - - - public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG - - @Test - public void testYarnRegistryAM() throws Throwable { - - - describe "create a masterless AM then perform YARN registry operations on it" - - - String clustername = createMiniCluster(configuration, 1, true) - - // get local binding - def registryOperations = microZKCluster.registryOperations - registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES) - - // verify the cluster has the YARN reg service live - def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry - assert rmRegistryService - - - - - - ServiceLauncher<SliderClient> launcher - launcher = createStandaloneAM(clustername, true, false) - SliderClient client = launcher.service - addToTeardown(client); - - ApplicationReport report = waitForClusterLive(client) - logReport(report) - List<ApplicationReport> apps = client.applications; - - List<ClusterNode> clusterNodes = client.listClusterNodesInRole( - SliderKeys.COMPONENT_AM) - assert ((List<ClusterNode>)clusterNodes).size() == 1 - - ClusterNode masterNode = clusterNodes[0] - log.info("Master node = ${masterNode}"); - - List<ClusterNode> nodes - String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM) - assert uuids.length == 1; - nodes = client.listClusterNodes(uuids); - assert ((List<ClusterNode>)nodes).size() == 1; - describe "AM Node UUID=${uuids[0]}" - - nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM) - assert ((List<ClusterNode>)nodes).size() == 1; - nodes = listNodesInRole(client, "") - assert ((List<ClusterNode>)nodes).size() == 1; - ClusterNode master = nodes[0] - assert master.role == SliderKeys.COMPONENT_AM - - - - - String username = client.username - def yarnRegistryClient = client.yarnAppListClient - describe("list of all applications") - logApplications(apps) - describe("apps of user $username") - List<ApplicationReport> userInstances = yarnRegistryClient.listInstances() - logApplications(userInstances) - assert userInstances.size() == 1 - describe("named app $clustername") - ApplicationReport instance = yarnRegistryClient.findInstance(clustername) - logReport(instance) - assert instance != null - - // sleep to allow registration to complete - sleep(5000) - - - - - try { - def yarnRegistryDump = client.dumpYarnRegistry(true).toString() - log.info("yarn service registry: \n${yarnRegistryDump}\n") - } catch (IOException ignored) { - - } - - - describe "service registry names" - def registryService = client.registryOperations - - def self = currentUser() - RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self)) - dumpArray(serviceTypes) - - def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE) - - def serviceRecords = RecordOperations.extractServiceRecords(registryService, - registryService.listDir(recordsPath)) - dumpCollection(serviceRecords) - assert serviceRecords.size() == 1 - - def serviceInstance = serviceRecords[0] - log.info(serviceInstance.toString()) - - assert 2 <= serviceInstance.external.size() - - // hit the registry web page - - def registryEndpoint = serviceInstance.getExternalEndpoint( - CustomRegistryConstants.REGISTRY_REST_API) - assert registryEndpoint != null - def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0] - describe("Registry WADL @ $registryURL") - - def publisherEndpoint = serviceInstance.getExternalEndpoint( - CustomRegistryConstants.PUBLISHER_REST_API) - - def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0] - def publisher = publisherURL.toString() - describe("Publisher") - - JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap) - def setlisting = GET(publisherURL) - - log.info(setlisting) - - UriMap uris = uriMapDeser.fromJson(setlisting) - assert uris.uris[RestPaths.SLIDER_CONFIGSET] - def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET) - JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>( - PublishedConfigSet) - def configSet = serDeser.fromJson(publishedJSON) - assert configSet.size() >= 1 - assert configSet.contains(ARTIFACT_NAME) - PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME) - - assert publishedYarnSite.empty - - //get the full URL - def yarnSitePublisher = appendToURL(publisher, - RestPaths.SLIDER_CONFIGSET, - ARTIFACT_NAME) - - String confJSON = GET(yarnSitePublisher) -// log.info(confJSON) - JsonSerDeser< PublishedConfiguration> confSerDeser = - new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration) - - publishedYarnSite = confSerDeser.fromJson(confJSON) - - assert !publishedYarnSite.empty - - - //get the XML - def yarnSiteXML = yarnSitePublisher + ".xml" - - - String confXML = GET(yarnSiteXML) - log.info("Conf XML at $yarnSiteXML = \n $confXML") - - String properties = GET(yarnSitePublisher + ".properties") - Properties parsedProps = new Properties() - parsedProps.load(new StringReader(properties)) - assert parsedProps.size() > 0 - def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS) - def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME) - assert rmAddrFromDownloadedProperties - assert rmHostnameFromDownloadedProperties - - String json = GET(yarnSitePublisher + ".json") - - - - describe("Registry List") - log.info(GET(registryURL)) - - - describe "Registry Retrieval Class" - // retrieval - - RegistryRetriever retriever = new RegistryRetriever(serviceInstance) - log.info retriever.toString() - - assert retriever.hasConfigurations(true) - PublishedConfigSet externalConfSet = retriever.getConfigurations(true) - dumpConfigurationSet(externalConfSet) - assert externalConfSet[ARTIFACT_NAME] - - - describe "verify SLIDER-52 processing" - def yarnSite = retriever.retrieveConfiguration( - externalConfSet, - ARTIFACT_NAME, - true) - assert !yarnSite.empty - def siteXML = yarnSite.asConfiguration() - def rmHostnameViaClientSideXML = parsedProps.get( - YarnConfiguration.RM_HOSTNAME) - assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties - def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS) - - log.info("RM from downloaded props = $rmAddrFromDownloadedProperties") - assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties - - describe "fetch missing artifact" - try { - retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true) - fail("expected a failure") - } catch (FileNotFoundException expected) { - // expected - } - describe "Internal configurations" - assert !retriever.hasConfigurations(false) - try { - retriever.getConfigurations(false) - fail("expected a failure") - } catch (FileNotFoundException expected) { - // expected - } - - - // retrieval via API - ActionRegistryArgs registryArgs = new ActionRegistryArgs() - registryArgs.verbose = true - - // list all - registryArgs.list = true; - describe registryArgs.toString() - client.actionRegistry(registryArgs) - - // list a named instance and expect a failure - registryArgs.list = true; - registryArgs.name = "unknown" - try { - client.actionRegistryListYarn(registryArgs) - } catch (PathNotFoundException expected) { - // expected - } - - // list all instances of an alternate type and expect failure - registryArgs.list = true; - registryArgs.name = null - registryArgs.serviceType = "org-apache-hadoop" - try { - client.actionRegistryListYarn(registryArgs) - } catch (PathNotFoundException expected) { - // expected - } - - registryArgs.serviceType = "" - - //set the name - registryArgs.name = clustername; - registryArgs.serviceType = SliderKeys.APP_TYPE - - - //now expect list to work - describe registryArgs.toString() - - def listedInstance = client.actionRegistryListYarn(registryArgs) - assert listedInstance[0].id == serviceInstance.id - - - // listconf - registryArgs.list = false; - registryArgs.listConf = true - describe registryArgs.toString() - - client.actionRegistry(registryArgs) - - // listconf --internal - registryArgs.list = false; - registryArgs.listConf = true - registryArgs.internal = true - describe registryArgs.toString() - assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) - - registryArgs.list = false; - registryArgs.listConf = false - registryArgs.internal = false - - def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG - registryArgs.getConf = yarn_site_config - - //properties format - registryArgs.format = "properties" - describe registryArgs.toString() - - client.actionRegistry(registryArgs) - - - File outputDir = new File("target/test_standalone_registry_am/output") - outputDir.mkdirs() - - // create a new registry args with the defaults back in - registryArgs = new ActionRegistryArgs(clustername) - registryArgs.getConf = yarn_site_config - registryArgs.dest = outputDir - describe registryArgs.toString() - client.actionRegistry(registryArgs) - assert new File(outputDir, yarn_site_config + ".xml").exists() - - registryArgs.format = "properties" - client.actionRegistry(registryArgs) - assert new File(outputDir, yarn_site_config + ".properties").exists() - - describe registryArgs.toString() - - def unknownFilename = "undefined-file" - registryArgs.getConf = unknownFilename - assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs) - - describe "stop cluster" - //now kill that cluster - assert 0 == clusterActionFreeze(client, clustername) - //list it & See if it is still there - ApplicationReport oldInstance = yarnRegistryClient.findInstance( - clustername) - assert oldInstance != null - assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED - - - - } -}
