Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts: slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b86aff8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b86aff8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b86aff8 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 0b86aff8de6910005b99f159d6807888895971ba Parents: 34396a0 52f1f6f Author: Steve Loughran <ste...@apache.org> Authored: Thu Aug 14 15:14:15 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Aug 14 15:24:15 2014 +0100 ---------------------------------------------------------------------- pom.xml | 9 +- slider-agent/pom.xml | 4 +- .../src/main/python/agent/AgentConfig.py | 2 + slider-agent/src/main/python/agent/Constants.py | 4 +- .../src/main/python/agent/Controller.py | 52 +- slider-agent/src/main/python/agent/Heartbeat.py | 6 +- slider-agent/src/main/python/agent/Registry.py | 57 + slider-agent/src/main/python/agent/main.py | 29 +- slider-agent/src/main/python/kazoo/__init__.py | 1 + slider-agent/src/main/python/kazoo/client.py | 1412 ++++++++++++++++++ .../src/main/python/kazoo/exceptions.py | 199 +++ .../src/main/python/kazoo/handlers/__init__.py | 1 + .../src/main/python/kazoo/handlers/gevent.py | 161 ++ .../src/main/python/kazoo/handlers/threading.py | 287 ++++ .../src/main/python/kazoo/handlers/utils.py | 93 ++ slider-agent/src/main/python/kazoo/hosts.py | 26 + .../src/main/python/kazoo/interfaces.py | 203 +++ .../src/main/python/kazoo/loggingsupport.py | 2 + .../src/main/python/kazoo/protocol/__init__.py | 1 + .../main/python/kazoo/protocol/connection.py | 623 ++++++++ .../src/main/python/kazoo/protocol/paths.py | 54 + .../main/python/kazoo/protocol/serialization.py | 396 +++++ .../src/main/python/kazoo/protocol/states.py | 237 +++ .../src/main/python/kazoo/recipe/__init__.py | 1 + .../src/main/python/kazoo/recipe/barrier.py | 214 +++ .../src/main/python/kazoo/recipe/counter.py | 94 ++ .../src/main/python/kazoo/recipe/election.py | 79 + .../src/main/python/kazoo/recipe/lock.py | 520 +++++++ .../src/main/python/kazoo/recipe/partitioner.py | 377 +++++ .../src/main/python/kazoo/recipe/party.py | 118 ++ .../src/main/python/kazoo/recipe/queue.py | 321 ++++ .../src/main/python/kazoo/recipe/watchers.py | 419 ++++++ slider-agent/src/main/python/kazoo/retry.py | 150 ++ slider-agent/src/main/python/kazoo/security.py | 138 ++ .../src/main/python/kazoo/testing/__init__.py | 5 + .../src/main/python/kazoo/testing/common.py | 283 ++++ .../src/main/python/kazoo/testing/harness.py | 180 +++ .../src/main/python/kazoo/tests/__init__.py | 0 .../src/main/python/kazoo/tests/test_barrier.py | 157 ++ .../src/main/python/kazoo/tests/test_build.py | 29 + .../src/main/python/kazoo/tests/test_client.py | 1098 ++++++++++++++ .../main/python/kazoo/tests/test_connection.py | 319 ++++ .../src/main/python/kazoo/tests/test_counter.py | 35 + .../main/python/kazoo/tests/test_election.py | 139 ++ .../main/python/kazoo/tests/test_exceptions.py | 22 + .../python/kazoo/tests/test_gevent_handler.py | 160 ++ .../src/main/python/kazoo/tests/test_lock.py | 517 +++++++ .../main/python/kazoo/tests/test_partitioner.py | 92 ++ .../src/main/python/kazoo/tests/test_party.py | 84 ++ .../src/main/python/kazoo/tests/test_paths.py | 98 ++ .../src/main/python/kazoo/tests/test_queue.py | 179 +++ .../src/main/python/kazoo/tests/test_retry.py | 77 + .../main/python/kazoo/tests/test_security.py | 40 + .../kazoo/tests/test_threading_handler.py | 326 ++++ .../main/python/kazoo/tests/test_watchers.py | 489 ++++++ .../src/main/python/kazoo/tests/util.py | 126 ++ .../src/test/python/agent/TestHeartbeat.py | 12 +- slider-agent/src/test/python/agent/TestMain.py | 16 +- slider-assembly/src/main/bash/README.md | 102 ++ slider-assembly/src/main/bash/slider-client.xml | 83 + slider-assembly/src/main/bash/slider_destroy | 64 + slider-assembly/src/main/bash/slider_setup | 173 +++ slider-assembly/src/main/bash/slider_setup.conf | 60 + slider-core/pom.xml | 46 +- .../api/ClusterDescriptionOperations.java | 12 +- .../org/apache/slider/api/InternalKeys.java | 137 ++ .../java/org/apache/slider/api/OptionKeys.java | 92 +- .../org/apache/slider/api/ResourceKeys.java | 62 + .../org/apache/slider/client/SliderClient.java | 17 +- .../apache/slider/common/SliderExitCodes.java | 2 +- .../org/apache/slider/common/SliderKeys.java | 5 + .../apache/slider/common/SliderXmlConfKeys.java | 13 - .../apache/slider/common/tools/SliderUtils.java | 23 +- .../slider/core/build/InstanceBuilder.java | 3 +- .../apache/slider/core/conf/MapOperations.java | 54 +- .../providers/AbstractProviderService.java | 18 + .../slider/providers/ProviderService.java | 22 + .../apache/slider/providers/ProviderUtils.java | 9 +- .../providers/agent/AgentClientProvider.java | 8 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 134 +- .../slideram/SliderAMClientProvider.java | 4 +- .../server/appmaster/SliderAppMaster.java | 231 ++- .../server/appmaster/actions/ActionHalt.java | 9 +- .../appmaster/actions/ActionKillContainer.java | 67 + .../appmaster/actions/ActionStartContainer.java | 15 +- .../appmaster/actions/ActionStopQueue.java | 15 +- .../appmaster/actions/ActionStopSlider.java | 15 +- .../server/appmaster/actions/AsyncAction.java | 16 +- .../actions/ProviderReportedContainerLoss.java | 7 +- .../actions/ProviderStartupCompleted.java | 7 +- .../server/appmaster/actions/QueueAccess.java | 31 +- .../server/appmaster/actions/QueueExecutor.java | 7 +- .../server/appmaster/actions/QueueService.java | 117 +- .../appmaster/actions/RenewingAction.java | 35 +- .../appmaster/actions/ResetFailureWindow.java | 39 + .../server/appmaster/monkey/ChaosEntry.java | 86 ++ .../server/appmaster/monkey/ChaosKillAM.java | 48 + .../appmaster/monkey/ChaosKillContainer.java | 75 + .../appmaster/monkey/ChaosMonkeyService.java | 86 ++ .../server/appmaster/monkey/ChaosTarget.java | 24 + .../appmaster/monkey/MonkeyPlayAction.java | 48 + .../slider/server/appmaster/state/AppState.java | 125 +- .../server/appmaster/state/RoleHistory.java | 2 +- .../server/appmaster/state/RoleInstance.java | 4 +- .../server/appmaster/state/RoleStatus.java | 24 +- .../appmaster/web/rest/agent/HeartBeat.java | 11 + .../services/workflow/LongLivedProcess.java | 7 +- .../workflow/WorkflowExecutorService.java | 2 +- .../TestFreezeThawMasterlessAM.groovy | 5 +- .../standalone/TestBuildStandaloneAM.groovy | 13 +- .../TestStandaloneAMMonkeyRestart.groovy | 94 ++ .../standalone/TestStandaloneAgentAM.groovy | 2 +- .../server/appmaster/actions/TestActions.groovy | 43 +- .../TestMockAppStateContainerFailure.groovy | 46 +- .../TestMockAppStateDynamicRoles.groovy | 2 +- .../TestMockAppStateFlexDynamicRoles.groovy | 2 +- .../appstate/TestMockAppStateFlexing.groovy | 2 +- .../TestMockAppStateRMOperations.groovy | 2 +- .../TestMockAppStateRebuildOnAMRestart.groovy | 2 +- .../TestMockAppStateRolePlacement.groovy | 2 +- .../appstate/TestMockAppStateRoleRelease.groovy | 2 +- .../TestMockContainerResourceAllocations.groovy | 2 +- .../model/mock/BaseMockAppStateTest.groovy | 22 +- .../model/mock/MockProviderService.groovy | 12 + .../model/mock/MockRMOperationHandler.groovy | 7 +- .../model/monkey/TestMockMonkey.groovy | 160 ++ .../apache/slider/test/SliderTestBase.groovy | 7 +- .../apache/slider/test/SliderTestUtils.groovy | 4 + .../agent/TestAgentProviderService.java | 35 +- slider-core/src/test/python/agent/main.py | 5 +- slider-core/src/test/resources/log4j.properties | 2 +- slider-install/README.md | 102 -- slider-install/pom.xml | 111 -- slider-install/src/main/bash/slider-client.xml | 83 - slider-install/src/main/bash/slider_destroy | 64 - slider-install/src/main/bash/slider_setup | 173 --- slider-install/src/main/bash/slider_setup.conf | 60 - .../accumulo/AccumuloClientProvider.java | 3 +- .../accumulo/AccumuloProviderService.java | 9 +- .../providers/hbase/HBaseClientProvider.java | 3 +- .../providers/hbase/HBaseProviderService.java | 4 +- .../failures/TestFailureThreshold.groovy | 150 -- .../TestRegionServerFailureThreshold.groovy | 185 +++ ...reezeReconfigureThawLiveRegionService.groovy | 4 +- .../src/test/resources/log4j.properties | 1 + 146 files changed, 13291 insertions(+), 1205 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 00f98cb,1337570..61f4d54 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@@ -33,11 -33,9 +33,12 @@@ import org.apache.hadoop.yarn.api.recor import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper; +import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; + import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.SliderClusterProtocol; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b6f1e49,7825e24..5ef32bf --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@@ -196,9 -198,13 +203,14 @@@ public class SliderAppMaster extends Ab public static final int HEARTBEAT_INTERVAL = 1000; public static final int NUM_RPC_HANDLERS = 5; + public static final String SLIDER_AM_RPC = "Slider AM RPC"; public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1; + /** + * Singleton of metrics registry + */ + public static final MetricRegistry metrics = new MetricRegistry(); + /** YARN RPC to communicate with the Resource Manager or Node Manager */ private YarnRPC yarnRPC; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy ---------------------------------------------------------------------- diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy index 711bc24,0665a9a..434f4c2 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy @@@ -64,11 -65,20 +65,17 @@@ class TestBuildStandaloneAM extends Age //but the cluster is still there for the default assert 0 == sliderClient.actionExists(clustername, false) - - - - // verify the YARN registry doesn't know of it - def serviceRegistryClient = sliderClient.YARNRegistryClient + + def serviceRegistryClient = sliderClient.yarnAppListClient ApplicationReport report = serviceRegistryClient.findInstance(clustername) assert report == null; - + + // verify that global resource options propagate from the CLI + def aggregateConf = sliderClient.loadPersistedClusterDescription(clustername) + def windowDays = aggregateConf.resourceOperations.globalOptions.getMandatoryOptionInt( + "yarn.container.failure.window.years") + assert 4 == windowDays + //and a second attempt will fail as the cluster now exists try { ServiceLauncher<SliderClient> cluster2 = createOrBuildCluster( http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --cc slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java index a4111fa,feffb08..747ccfa --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java @@@ -22,11 -22,9 +22,11 @@@ package org.apache.slider.providers.hba import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.registry.client.api.RegistryWriter; +import org.apache.hadoop.yarn.registry.client.types.ServiceEntry; + import org.apache.slider.api.InternalKeys; import org.apache.slider.common.SliderKeys; import org.apache.slider.api.ClusterDescription; - import org.apache.slider.api.OptionKeys; import org.apache.slider.api.RoleKeys; import org.apache.slider.api.StatusKeys; import org.apache.slider.core.conf.AggregateConf;