Repository: incubator-slider Updated Branches: refs/heads/develop 7e667fc52 -> 8d319482b
SLIDER-608 AM to only register service record at app level Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d319482 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d319482 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d319482 Branch: refs/heads/develop Commit: 8d319482bd33238dde0c688512812f2acb72d23d Parents: 7e667fc Author: Steve Loughran <ste...@apache.org> Authored: Mon Nov 3 17:33:28 2014 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Nov 3 17:33:28 2014 +0000 ---------------------------------------------------------------------- .../server/appmaster/SliderAppMaster.java | 11 --- .../TestStandaloneYarnRegistryAM.groovy | 40 +++++++--- .../org/apache/slider/test/Outcome.groovy | 47 ++++++++++++ .../apache/slider/test/SliderTestUtils.groovy | 52 +++++++++++++ .../funtest/framework/CommandTestBase.groovy | 81 +------------------- .../funtest/lifecycle/AgentRegistryIT.groovy | 21 +++++ 6 files changed, 152 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index a3c9b95..8791d19 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1093,17 +1093,6 @@ the registry with/without the new record format yarnRegistryOperations.getSelfRegistrationPath(), true); } - - // and a shorter lived binding to the app - String attempt = appAttemptID.toString(); - ServiceRecord attemptRecord = new ServiceRecord(serviceRecord); - attemptRecord.set(YarnRegistryAttributes.YARN_ID, attempt); - attemptRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, - PersistencePolicies.APPLICATION_ATTEMPT); - yarnRegistryOperations.putComponent( - RegistryPathUtils.encodeYarnID(attempt), - serviceRecord); - } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy index c5e7190..a16d3d0 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy @@ -34,6 +34,7 @@ import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.ActionResolveArgs import org.apache.slider.common.params.Arguments +import org.apache.slider.core.exceptions.NotFoundException import org.apache.slider.core.main.LauncherExitCodes import static org.apache.hadoop.registry.client.binding.RegistryUtils.* @@ -47,10 +48,10 @@ import org.apache.slider.core.persist.JsonSerDeser import org.apache.slider.core.registry.docstore.PublishedConfigSet import org.apache.slider.core.registry.docstore.PublishedConfiguration import org.apache.slider.core.registry.docstore.UriMap -import org.apache.slider.core.registry.info.CustomRegistryConstants import org.apache.slider.core.registry.retrieve.RegistryRetriever import org.apache.slider.server.appmaster.PublishedArtifacts import org.apache.slider.server.appmaster.web.rest.RestPaths +import org.apache.slider.test.Outcome; import org.junit.Test import static org.apache.slider.core.registry.info.CustomRegistryConstants.* @@ -102,13 +103,13 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM) assert uuids.length == 1; nodes = client.listClusterNodes(uuids); - assert ((List<ClusterNode>)nodes).size() == 1; + assert nodes.size() == 1; describe "AM Node UUID=${uuids[0]}" nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM) - assert ((List<ClusterNode>)nodes).size() == 1; + assert nodes.size() == 1; nodes = listNodesInRole(client, "") - assert ((List<ClusterNode>)nodes).size() == 1; + assert nodes.size() == 1; ClusterNode master = nodes[0] assert master.role == SliderKeys.COMPONENT_AM @@ -163,10 +164,12 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != "" def externalEndpoints = serviceRecord.external; assert externalEndpoints.size() > 0 +/* def am_ipc_protocol = AM_IPC_PROTOCOL serviceRecord.getExternalEndpoint(am_ipc_protocol) assert null != am_ipc_protocol; +*/ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API) assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) @@ -273,8 +276,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { list: true)) // hit the registry web page - def registryEndpoint = serviceRecord.getExternalEndpoint( - CustomRegistryConstants.REGISTRY_REST_API) + def registryEndpoint = serviceRecord.getExternalEndpoint(REGISTRY_REST_API) assert registryEndpoint != null def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0] @@ -290,8 +292,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { // Look at the Registry WADL describe("Registry WADL @ $registryURL") - def publisherEndpoint = serviceRecord.getExternalEndpoint( - CustomRegistryConstants.PUBLISHER_REST_API) + def publisherEndpoint = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API) def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0] def publisher = publisherURL.toString() @@ -348,7 +349,6 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { String json = GET(yarnSitePublisher + ".json") - describe("Registry List") log.info(GET(registryURL)) @@ -508,7 +508,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { assert oldInstance != null assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED - + // verify hbase to path generation filters things def hbase = homePathForUser(HBASE) def hbaseServices = serviceclassPath(hbase, SliderKeys.APP_TYPE) @@ -529,5 +529,25 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase { } + // now expect the AM to have had its service record deleted + ActionResolveArgs finalResolve = new ActionResolveArgs( + path: instanceRecordPath) + + repeatUntilSuccess(this.&probeForEntryMissing, 10000, 1000, + [client:client, + resolve:finalResolve], + true, + "registry entry never deleted") {} + } + + Outcome probeForEntryMissing(Map args) { + SliderClient client = (SliderClient)args["client"] + ActionResolveArgs resolveArgs = (ActionResolveArgs) args["resolve"] + try { + client.actionResolve(resolveArgs) + return Outcome.Retry + } catch (NotFoundException ignored) { + return Outcome.Success + } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy b/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy new file mode 100644 index 0000000..d7e027f --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.test + +/** + * Outcome for probes + */ + +class Outcome { + + public final String name; + + private Outcome(String name) { + this.name = name + } + + static Outcome Success = new Outcome( + "Success") + static Outcome Retry = new Outcome("Retry") + static Outcome Fail = new Outcome("Fail") + + /** + * build from a bool, where false is mapped to retry + * @param b boolean + * @return an outcome + */ + static Outcome fromBool(boolean b) { + return b ? Success : Retry; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 8bf1959..1afbafc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -925,4 +925,56 @@ class SliderTestUtils extends Assert { throw new FileNotFoundException("$text: $file not found in $builder") } } + + /** + * Repeat a probe until it succeeds, if it does not execute a failure + * closure then raise an exception with the supplied message + * @param probe probe + * @param timeout time in millis before giving up + * @param sleepDur sleep between failing attempts + * @param args map of arguments to the probe + * @param failIfUnsuccessful if the probe fails after all the attempts + * âshould it raise an exception + * @param failureMessage message to include in exception raised + * @param failureHandler closure to invoke prior to the failure being raised + */ + protected void repeatUntilSuccess(Closure probe, + int timeout, int sleepDur, + Map args, + boolean failIfUnsuccessful, + String failureMessage, + Closure failureHandler) { + int attemptCount = 0 + boolean succeeded = false; + boolean completed = false; + Duration duration = new Duration(timeout) + duration.start(); + while (!completed) { + Outcome outcome = (Outcome) probe(args) + if (outcome.equals(Outcome.Success)) { + // success + log.debug("Success after $attemptCount attempt(s)") + succeeded = true; + completed = true; + } else if (outcome.equals(Outcome.Retry)) { + // failed but retry possible + attemptCount++; + completed = duration.limitExceeded + if (!completed) { + sleep(sleepDur) + } + } else if (outcome.equals(Outcome.Fail)) { + // fast fail + completed = true; + } + } + + if (failIfUnsuccessful & !succeeded) { + if (failureHandler) { + failureHandler() + } + fail(failureMessage) + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 640deaf..36515c5 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.StatusKeys import org.apache.slider.common.tools.ConfigHelper -import org.apache.slider.common.tools.Duration import org.apache.slider.core.launch.SerializedApplicationReport import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys @@ -40,6 +39,8 @@ import org.apache.slider.common.tools.SliderUtils import org.apache.slider.client.SliderClient import org.apache.slider.core.persist.ApplicationReportSerDeser import org.apache.slider.test.SliderTestUtils +import org.apache.slider.test.Outcome; + import org.junit.Before import org.junit.BeforeClass import org.junit.Rule @@ -934,84 +935,6 @@ abstract class CommandTestBase extends SliderTestUtils { } } - /** - * Outcome for probes - */ - static class Outcome { - - public final String name; - - private Outcome(String name) { - this.name = name - } - - static Outcome Success = new Outcome("Success") - static Outcome Retry = new Outcome("Retry") - static Outcome Fail = new Outcome("Fail") - - - /** - * build from a bool, where false is mapped to retry - * @param b boolean - * @return an outcome - */ - static Outcome fromBool(boolean b) { - return b? Success: Retry; - } - - } - - /** - * Repeat a probe until it succeeds, if it does not execute a failure - * closure then raise an exception with the supplied message - * @param probe probe - * @param timeout time in millis before giving up - * @param sleepDur sleep between failing attempts - * @param args map of arguments to the probe - * @param failIfUnsuccessful if the probe fails after all the attempts - * âshould it raise an exception - * @param failureMessage message to include in exception raised - * @param failureHandler closure to invoke prior to the failure being raised - */ - protected void repeatUntilSuccess(Closure probe, - int timeout, int sleepDur, - Map args, - boolean failIfUnsuccessful, - String failureMessage, - Closure failureHandler) { - int attemptCount = 0 - boolean succeeded = false; - boolean completed = false; - Duration duration = new Duration(timeout) - duration.start(); - while (!completed) { - Outcome outcome = (Outcome) probe(args) - if (outcome.equals(Outcome.Success)) { - // success - log.debug("Success after $attemptCount attempt(s)") - succeeded = true; - completed = true; - } else if (outcome.equals(Outcome.Retry)) { - // failed but retry possible - attemptCount++; - completed = duration.limitExceeded - if (!completed) { - sleep(sleepDur) - } - } else if (outcome.equals(Outcome.Fail)) { - // fast fail - completed = true; - } - } - - if (failIfUnsuccessful & !succeeded) { - if (failureHandler) { - failureHandler() - } - fail(failureMessage) - } - } - public String getInfoAmWebUrl(String applicationName) { ClusterDescription cd = execStatus(applicationName); String urlString = cd.getInfo("info.am.web.url"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d319482/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy index f23b16e..fe63660 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy @@ -24,10 +24,15 @@ import org.apache.hadoop.registry.client.binding.RegistryUtils import org.apache.hadoop.registry.client.types.Endpoint import org.apache.hadoop.registry.client.types.ServiceRecord import org.apache.hadoop.yarn.api.records.YarnApplicationState +import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.ActionResolveArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.NotFoundException +import org.apache.slider.test.Outcome + import static org.apache.slider.core.registry.info.CustomRegistryConstants.* import org.apache.slider.funtest.framework.AgentCommandTestBase import org.apache.slider.funtest.framework.FuntestProperties @@ -136,6 +141,15 @@ public class AgentRegistryIT extends AgentCommandTestBase //cluster now missing exists(EXIT_UNKNOWN_INSTANCE, CLUSTER) + + + repeatUntilSuccess(this.&probeForEntryMissing, 10000, 1000, + [path: appPath], + true, + "registry entry never deleted") { + // await the registry lookup to fail + resolve(EXIT_NOT_FOUND, [ARG_PATH, appPath]) + } } /** @@ -145,4 +159,11 @@ public class AgentRegistryIT extends AgentCommandTestBase public String homepath() { return RegistryUtils.homePathForCurrentUser() } + + + Outcome probeForEntryMissing(Map args) { + String path = args["path"] + def shell = slider([ACTION_RESOLVE, ARG_PATH, path]) + return Outcome.fromBool(shell.ret == EXIT_NOT_FOUND) + } }
