Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts: slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c51f2f0e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c51f2f0e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c51f2f0e Branch: refs/heads/develop Commit: c51f2f0eac6a7938e1f0acc27a4f5df90bb88339 Parents: 733745e 70b4b75 Author: Steve Loughran <ste...@apache.org> Authored: Sun Oct 5 19:56:47 2014 -0700 Committer: Steve Loughran <ste...@apache.org> Committed: Sun Oct 5 19:56:47 2014 -0700 ---------------------------------------------------------------------- .../accumulo/configuration/accumulo-site.xml | 19 +- app-packages/accumulo/configuration/global.xml | 7 +- app-packages/accumulo/pom.xml | 2 +- .../funtest/accumulo/AccumuloBasicIT.groovy | 4 +- .../slider/funtest/accumulo/CertUtil.java | 50 +++- .../app-pkg-template/appConfig-default.json | 2 +- .../command-logger/slider-pkg/appConfig.json | 2 +- app-packages/hbase/appConfig-default.json | 2 +- app-packages/hbase/pom.xml | 2 +- app-packages/memcached/appConfig-default.json | 2 +- app-packages/storm-win/appConfig-default.json | 2 - app-packages/storm-win/metainfo.xml | 12 +- .../storm-win/package/scripts/params.py | 2 - .../storm-win/package/scripts/service.py | 1 + .../storm-win/package/scripts/status_params.py | 2 - .../storm-win/package/templates/config.yaml.j2 | 9 - app-packages/storm/appConfig-default.json | 4 +- app-packages/storm/metainfo.xml | 23 +- app-packages/storm/package/scripts/params.py | 2 - .../storm/package/templates/config.yaml.j2 | 9 - app-packages/storm/resources-default.json | 4 - pom.xml | 2 + slider-assembly/src/main/scripts/slider | 2 +- .../org/apache/slider/client/SliderClient.java | 77 +++-- .../apache/slider/common/SliderXmlConfKeys.java | 4 +- .../apache/slider/core/conf/AggregateConf.java | 20 ++ .../apache/slider/core/conf/MapOperations.java | 16 +- .../slider/core/exceptions/SliderException.java | 6 +- .../TriggerClusterTeardownException.java | 16 +- .../providers/AbstractClientProvider.java | 9 +- .../providers/AbstractProviderService.java | 21 +- .../apache/slider/providers/ProviderCore.java | 9 +- .../apache/slider/providers/ProviderUtils.java | 6 +- .../providers/agent/AgentProviderService.java | 28 +- .../providers/agent/ComponentInstanceState.java | 2 +- .../slideram/SliderAMClientProvider.java | 28 ++ .../slideram/SliderAMProviderService.java | 8 - .../apache/slider/server/appmaster/AMUtils.java | 5 +- .../server/appmaster/SliderAppMaster.java | 298 +++++++++++-------- .../appmaster/actions/ActionStopQueue.java | 8 +- .../appmaster/actions/ActionStopSlider.java | 116 +++++++- .../server/appmaster/actions/AsyncAction.java | 2 +- .../server/appmaster/actions/QueueExecutor.java | 10 + .../server/appmaster/actions/QueueService.java | 22 +- .../slider/server/appmaster/state/AppState.java | 13 +- .../server/appmaster/state/RoleStatus.java | 20 ++ .../workflow/WorkflowSequenceService.java | 8 +- .../agent/actions/TestActionStatus.groovy | 16 +- .../TestFreezeThawFlexStandaloneAM.groovy | 99 ++++++ .../TestFreezeThawMasterlessAM.groovy | 92 ------ .../standalone/TestBuildStandaloneAM.groovy | 22 +- .../standalone/TestStandaloneAMDestroy.groovy | 20 +- .../TestStandaloneAMMonkeyRestart.groovy | 12 +- .../standalone/TestStandaloneAMRestart.groovy | 77 ++++- .../standalone/TestStandaloneAgentAM.groovy | 2 +- .../slider/providers/agent/TestAgentEcho.groovy | 18 +- .../TestMockAppStateContainerFailure.groovy | 1 - .../slider/test/YarnMiniClusterTestBase.groovy | 37 ++- .../accumulo/live/TestAccFreezeThaw.groovy | 7 +- .../providers/hbase/HBaseProviderService.java | 23 -- .../build/TestBuildThawClusterM1W1.groovy | 3 +- 61 files changed, 857 insertions(+), 490 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy ---------------------------------------------------------------------- diff --cc app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy index 7ddaf9f,5ebce0a..fd8af9a --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy @@@ -167,11 -167,13 +167,12 @@@ class AccumuloBasicIT extends AccumuloA public static String getMonitorUrl(SliderClient sliderClient, String clusterName) { int tries = 5 + Exception caught; while (true) { try { - CuratorServiceInstance<ServiceInstanceData> instance = - sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE, clusterName) - ServiceInstanceData serviceInstanceData = instance.payload - RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData) + ServiceRecord instance = + sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName) + RegistryRetriever retriever = new RegistryRetriever(instance) PublishedConfiguration configuration = retriever.retrieveConfiguration( retriever.getConfigurations(true), "quicklinks", true) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 2625d74,5676f3f..a259e42 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@@ -50,14 -51,9 +51,15 @@@ import org.apache.hadoop.yarn.client.ap import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; + import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.registry.client.api.RegistryOperations; +import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies; +import org.apache.hadoop.yarn.registry.client.types.ServiceRecord; +import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.util.ConverterUtils; @@@ -357,8 -358,8 +362,9 @@@ public class SliderAppMaster extends Ab private final QueueService actionQueues = new QueueService(); private String agentOpsUrl; private String agentStatusUrl; + private YarnRegistryViewForProviders yarnRegistryOperations; private FsDelegationTokenManager fsDelegationTokenManager; + private RegisterApplicationMasterResponse amRegistrationData; /** * Service Constructor http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy ---------------------------------------------------------------------- diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy index 13c876e,a14a14d..b6ac657 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy @@@ -30,6 -30,6 +30,7 @@@ import org.apache.slider.common.params. import org.apache.slider.core.exceptions.SliderException import org.apache.slider.core.main.LauncherExitCodes import org.apache.slider.core.main.ServiceLauncher ++import org.apache.slider.core.registry.YarnAppListClient import org.junit.Test import static org.apache.slider.common.params.Arguments.ARG_COMP_OPT @@@ -66,7 -66,9 +67,8 @@@ class TestBuildStandaloneAM extends Age //but the cluster is still there for the default assert 0 == sliderClient.actionExists(clustername, false) - def serviceRegistryClient = sliderClient.yarnAppListClient - + // verify the YARN registry doesn't know of it - def serviceRegistryClient = sliderClient.YARNRegistryClient ++ YarnAppListClient serviceRegistryClient = sliderClient.yarnAppListClient ApplicationReport report = serviceRegistryClient.findInstance(clustername) assert report == null; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy ----------------------------------------------------------------------