com.ebay => org.apache This closes: #22 Review: #22
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/101bcad3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/101bcad3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/101bcad3 Branch: refs/heads/master Commit: 101bcad35c931fcac788edf0712e88e13bf7789a Parents: 67ecf06 Author: klucar <klu...@gmail.com> Authored: Wed Oct 28 09:54:48 2015 -0400 Committer: Santosh Marella <mare...@gmail.com> Committed: Wed Oct 28 08:58:47 2015 -0700 ---------------------------------------------------------------------- build.gradle | 4 +- config/findbugs/excludeFilter.xml | 2 +- docs/myriad-dev.md | 6 +- docs/samples/yarn-site.xml | 4 +- docs/vagrant.md | 4 +- .../config/findbugs/excludeFilter.xml | 2 +- .../executor/ContainerTaskStatusRequest.java | 45 -- .../myriad/executor/MyriadExecutorDefaults.java | 75 --- .../com/ebay/myriad/executor/NMTaskConfig.java | 119 ---- .../executor/ContainerTaskStatusRequest.java | 45 ++ .../myriad/executor/MyriadExecutorDefaults.java | 75 +++ .../apache/myriad/executor/NMTaskConfig.java | 119 ++++ myriad-executor/build.gradle | 2 +- .../config/findbugs/excludeFilter.xml | 2 +- .../ebay/myriad/executor/MyriadExecutor.java | 115 ---- .../executor/MyriadExecutorAuxService.java | 116 ---- .../apache/myriad/executor/MyriadExecutor.java | 115 ++++ .../executor/MyriadExecutorAuxService.java | 116 ++++ .../config/findbugs/excludeFilter.xml | 2 +- .../java/com/ebay/myriad/DisruptorManager.java | 148 ----- .../src/main/java/com/ebay/myriad/Main.java | 299 ---------- .../main/java/com/ebay/myriad/MesosModule.java | 107 ---- .../main/java/com/ebay/myriad/MyriadModule.java | 165 ------ .../com/ebay/myriad/api/ClustersResource.java | 287 ---------- .../ebay/myriad/api/ConfigurationResource.java | 48 -- .../ebay/myriad/api/SchedulerStateResource.java | 69 --- .../api/model/FlexDownClusterRequest.java | 75 --- .../api/model/FlexDownServiceRequest.java | 63 --- .../myriad/api/model/FlexUpClusterRequest.java | 74 --- .../myriad/api/model/FlexUpServiceRequest.java | 61 --- .../api/model/GetSchedulerStateResponse.java | 59 -- .../MyriadBadConfigurationException.java | 31 -- .../configuration/MyriadConfiguration.java | 259 --------- .../MyriadExecutorConfiguration.java | 59 -- .../configuration/NodeManagerConfiguration.java | 94 ---- .../configuration/OptionalSerializer.java | 110 ---- .../configuration/ServiceConfiguration.java | 142 ----- .../ebay/myriad/health/HealthCheckUtils.java | 45 -- .../myriad/health/MesosDriverHealthCheck.java | 49 -- .../myriad/health/MesosMasterHealthCheck.java | 88 --- .../myriad/health/ZookeeperHealthCheck.java | 43 -- .../myriad/policy/LeastAMNodesFirstPolicy.java | 166 ------ .../ebay/myriad/policy/NodeScaleDownPolicy.java | 37 -- .../scheduler/DownloadNMExecutorCLGenImpl.java | 97 ---- .../scheduler/ExecutorCommandLineGenerator.java | 29 - .../scheduler/ExtendedResourceProfile.java | 80 --- .../com/ebay/myriad/scheduler/MyriadDriver.java | 66 --- .../myriad/scheduler/MyriadDriverManager.java | 103 ---- .../ebay/myriad/scheduler/MyriadOperations.java | 236 -------- .../ebay/myriad/scheduler/MyriadScheduler.java | 165 ------ .../myriad/scheduler/NMExecutorCLGenImpl.java | 180 ------ .../java/com/ebay/myriad/scheduler/NMPorts.java | 78 --- .../com/ebay/myriad/scheduler/NMProfile.java | 63 --- .../ebay/myriad/scheduler/NMProfileManager.java | 54 -- .../scheduler/NMTaskFactoryAnnotation.java | 36 -- .../java/com/ebay/myriad/scheduler/Ports.java | 26 - .../com/ebay/myriad/scheduler/Rebalancer.java | 104 ---- .../ebay/myriad/scheduler/ReconcileService.java | 85 --- .../ebay/myriad/scheduler/SchedulerUtils.java | 73 --- .../scheduler/ServiceCommandLineGenerator.java | 40 -- .../myriad/scheduler/ServiceProfileManager.java | 55 -- .../scheduler/ServiceResourceProfile.java | 131 ----- .../scheduler/ServiceTaskConstraints.java | 53 -- .../scheduler/ServiceTaskFactoryImpl.java | 239 -------- .../ebay/myriad/scheduler/TaskConstraints.java | 35 -- .../scheduler/TaskConstraintsManager.java | 48 -- .../com/ebay/myriad/scheduler/TaskFactory.java | 201 ------- .../ebay/myriad/scheduler/TaskTerminator.java | 86 --- .../com/ebay/myriad/scheduler/TaskUtils.java | 209 ------- .../scheduler/constraints/Constraint.java | 35 -- .../constraints/ConstraintFactory.java | 36 -- .../scheduler/constraints/LikeConstraint.java | 113 ---- .../scheduler/event/DisconnectedEvent.java | 37 -- .../event/DisconnectedEventFactory.java | 33 -- .../ebay/myriad/scheduler/event/ErrorEvent.java | 46 -- .../scheduler/event/ErrorEventFactory.java | 33 -- .../scheduler/event/ExecutorLostEvent.java | 65 --- .../event/ExecutorLostEventFactory.java | 33 -- .../scheduler/event/FrameworkMessageEvent.java | 65 --- .../event/FrameworkMessageEventFactory.java | 33 -- .../scheduler/event/OfferRescindedEvent.java | 47 -- .../event/OfferRescindedEventFactory.java | 32 -- .../scheduler/event/ReRegisteredEvent.java | 46 -- .../event/ReRegisteredEventFactory.java | 32 -- .../myriad/scheduler/event/RegisteredEvent.java | 56 -- .../scheduler/event/RegisteredEventFactory.java | 33 -- .../scheduler/event/ResourceOffersEvent.java | 48 -- .../event/ResourceOffersEventFactory.java | 32 -- .../myriad/scheduler/event/SlaveLostEvent.java | 46 -- .../scheduler/event/SlaveLostEventFactory.java | 33 -- .../scheduler/event/StatusUpdateEvent.java | 47 -- .../event/StatusUpdateEventFactory.java | 33 -- .../handlers/DisconnectedEventHandler.java | 37 -- .../event/handlers/ErrorEventHandler.java | 38 -- .../handlers/ExecutorLostEventHandler.java | 42 -- .../handlers/FrameworkMessageEventHandler.java | 41 -- .../handlers/OfferRescindedEventHandler.java | 37 -- .../handlers/ReRegisteredEventHandler.java | 46 -- .../event/handlers/RegisteredEventHandler.java | 49 -- .../handlers/ResourceOffersEventHandler.java | 303 ---------- .../event/handlers/SlaveLostEventHandler.java | 39 -- .../handlers/StatusUpdateEventHandler.java | 97 ---- .../myriad/scheduler/fgs/ConsumedOffer.java | 55 -- .../scheduler/fgs/NMHeartBeatHandler.java | 168 ------ .../com/ebay/myriad/scheduler/fgs/Node.java | 89 --- .../ebay/myriad/scheduler/fgs/NodeStore.java | 54 -- .../ebay/myriad/scheduler/fgs/OfferFeed.java | 46 -- .../scheduler/fgs/OfferLifecycleManager.java | 115 ---- .../ebay/myriad/scheduler/fgs/OfferUtils.java | 53 -- .../scheduler/fgs/YarnNodeCapacityManager.java | 223 -------- .../scheduler/yarn/MyriadCapacityScheduler.java | 86 --- .../scheduler/yarn/MyriadFairScheduler.java | 86 --- .../scheduler/yarn/MyriadFifoScheduler.java | 86 --- .../scheduler/yarn/RMNodeEventHandler.java | 43 -- .../yarn/interceptor/BaseInterceptor.java | 68 --- .../yarn/interceptor/CompositeInterceptor.java | 139 ----- .../yarn/interceptor/InterceptorRegistry.java | 28 - .../MyriadInitializationInterceptor.java | 58 -- .../interceptor/YarnSchedulerInterceptor.java | 95 ---- .../java/com/ebay/myriad/state/Cluster.java | 99 ---- .../java/com/ebay/myriad/state/MyriadState.java | 56 -- .../com/ebay/myriad/state/MyriadStateStore.java | 33 -- .../java/com/ebay/myriad/state/NodeTask.java | 121 ---- .../com/ebay/myriad/state/SchedulerState.java | 549 ------------------- .../myriad/state/utils/ByteBufferSupport.java | 368 ------------- .../ebay/myriad/state/utils/StoreContext.java | 278 ---------- .../myriad/webapp/HttpConnectorProvider.java | 49 -- .../ebay/myriad/webapp/MyriadServletModule.java | 45 -- .../com/ebay/myriad/webapp/MyriadWebServer.java | 72 --- .../ebay/myriad/webapp/WebAppGuiceModule.java | 34 -- .../recovery/MyriadFileSystemRMStateStore.java | 4 +- .../org/apache/myriad/DisruptorManager.java | 147 +++++ .../src/main/java/org/apache/myriad/Main.java | 286 ++++++++++ .../java/org/apache/myriad/MesosModule.java | 107 ++++ .../java/org/apache/myriad/MyriadModule.java | 164 ++++++ .../org/apache/myriad/api/ClustersResource.java | 286 ++++++++++ .../myriad/api/ConfigurationResource.java | 48 ++ .../myriad/api/SchedulerStateResource.java | 68 +++ .../api/model/FlexDownClusterRequest.java | 75 +++ .../api/model/FlexDownServiceRequest.java | 63 +++ .../myriad/api/model/FlexUpClusterRequest.java | 74 +++ .../myriad/api/model/FlexUpServiceRequest.java | 61 +++ .../api/model/GetSchedulerStateResponse.java | 59 ++ .../MyriadBadConfigurationException.java | 31 ++ .../configuration/MyriadConfiguration.java | 257 +++++++++ .../MyriadExecutorConfiguration.java | 59 ++ .../configuration/NodeManagerConfiguration.java | 94 ++++ .../configuration/OptionalSerializer.java | 110 ++++ .../configuration/ServiceConfiguration.java | 138 +++++ .../apache/myriad/health/HealthCheckUtils.java | 45 ++ .../myriad/health/MesosDriverHealthCheck.java | 48 ++ .../myriad/health/MesosMasterHealthCheck.java | 88 +++ .../myriad/health/ZookeeperHealthCheck.java | 43 ++ .../myriad/policy/LeastAMNodesFirstPolicy.java | 164 ++++++ .../myriad/policy/NodeScaleDownPolicy.java | 37 ++ .../scheduler/DownloadNMExecutorCLGenImpl.java | 97 ++++ .../scheduler/ExecutorCommandLineGenerator.java | 29 + .../scheduler/ExtendedResourceProfile.java | 80 +++ .../apache/myriad/scheduler/MyriadDriver.java | 66 +++ .../myriad/scheduler/MyriadDriverManager.java | 103 ++++ .../myriad/scheduler/MyriadOperations.java | 236 ++++++++ .../myriad/scheduler/MyriadScheduler.java | 161 ++++++ .../myriad/scheduler/NMExecutorCLGenImpl.java | 180 ++++++ .../org/apache/myriad/scheduler/NMPorts.java | 78 +++ .../org/apache/myriad/scheduler/NMProfile.java | 63 +++ .../myriad/scheduler/NMProfileManager.java | 54 ++ .../scheduler/NMTaskFactoryAnnotation.java | 36 ++ .../java/org/apache/myriad/scheduler/Ports.java | 26 + .../org/apache/myriad/scheduler/Rebalancer.java | 101 ++++ .../myriad/scheduler/ReconcileService.java | 84 +++ .../apache/myriad/scheduler/SchedulerUtils.java | 73 +++ .../scheduler/ServiceCommandLineGenerator.java | 40 ++ .../myriad/scheduler/ServiceProfileManager.java | 55 ++ .../scheduler/ServiceResourceProfile.java | 131 +++++ .../scheduler/ServiceTaskConstraints.java | 53 ++ .../scheduler/ServiceTaskFactoryImpl.java | 239 ++++++++ .../myriad/scheduler/TaskConstraints.java | 35 ++ .../scheduler/TaskConstraintsManager.java | 48 ++ .../apache/myriad/scheduler/TaskFactory.java | 200 +++++++ .../apache/myriad/scheduler/TaskTerminator.java | 86 +++ .../org/apache/myriad/scheduler/TaskUtils.java | 208 +++++++ .../scheduler/constraints/Constraint.java | 35 ++ .../constraints/ConstraintFactory.java | 36 ++ .../scheduler/constraints/LikeConstraint.java | 113 ++++ .../scheduler/event/DisconnectedEvent.java | 37 ++ .../event/DisconnectedEventFactory.java | 33 ++ .../myriad/scheduler/event/ErrorEvent.java | 46 ++ .../scheduler/event/ErrorEventFactory.java | 33 ++ .../scheduler/event/ExecutorLostEvent.java | 65 +++ .../event/ExecutorLostEventFactory.java | 33 ++ .../scheduler/event/FrameworkMessageEvent.java | 65 +++ .../event/FrameworkMessageEventFactory.java | 33 ++ .../scheduler/event/OfferRescindedEvent.java | 47 ++ .../event/OfferRescindedEventFactory.java | 32 ++ .../scheduler/event/ReRegisteredEvent.java | 46 ++ .../event/ReRegisteredEventFactory.java | 32 ++ .../myriad/scheduler/event/RegisteredEvent.java | 56 ++ .../scheduler/event/RegisteredEventFactory.java | 33 ++ .../scheduler/event/ResourceOffersEvent.java | 48 ++ .../event/ResourceOffersEventFactory.java | 32 ++ .../myriad/scheduler/event/SlaveLostEvent.java | 46 ++ .../scheduler/event/SlaveLostEventFactory.java | 33 ++ .../scheduler/event/StatusUpdateEvent.java | 47 ++ .../event/StatusUpdateEventFactory.java | 33 ++ .../handlers/DisconnectedEventHandler.java | 36 ++ .../event/handlers/ErrorEventHandler.java | 37 ++ .../handlers/ExecutorLostEventHandler.java | 41 ++ .../handlers/FrameworkMessageEventHandler.java | 40 ++ .../handlers/OfferRescindedEventHandler.java | 36 ++ .../handlers/ReRegisteredEventHandler.java | 46 ++ .../event/handlers/RegisteredEventHandler.java | 46 ++ .../handlers/ResourceOffersEventHandler.java | 291 ++++++++++ .../event/handlers/SlaveLostEventHandler.java | 39 ++ .../handlers/StatusUpdateEventHandler.java | 97 ++++ .../myriad/scheduler/fgs/ConsumedOffer.java | 55 ++ .../scheduler/fgs/NMHeartBeatHandler.java | 163 ++++++ .../org/apache/myriad/scheduler/fgs/Node.java | 89 +++ .../apache/myriad/scheduler/fgs/NodeStore.java | 54 ++ .../apache/myriad/scheduler/fgs/OfferFeed.java | 46 ++ .../scheduler/fgs/OfferLifecycleManager.java | 114 ++++ .../apache/myriad/scheduler/fgs/OfferUtils.java | 53 ++ .../scheduler/fgs/YarnNodeCapacityManager.java | 219 ++++++++ .../scheduler/yarn/MyriadCapacityScheduler.java | 86 +++ .../scheduler/yarn/MyriadFairScheduler.java | 86 +++ .../scheduler/yarn/MyriadFifoScheduler.java | 85 +++ .../scheduler/yarn/RMNodeEventHandler.java | 43 ++ .../yarn/interceptor/BaseInterceptor.java | 68 +++ .../yarn/interceptor/CompositeInterceptor.java | 139 +++++ .../yarn/interceptor/InterceptorRegistry.java | 28 + .../MyriadInitializationInterceptor.java | 58 ++ .../interceptor/YarnSchedulerInterceptor.java | 95 ++++ .../java/org/apache/myriad/state/Cluster.java | 99 ++++ .../org/apache/myriad/state/MyriadState.java | 56 ++ .../apache/myriad/state/MyriadStateStore.java | 33 ++ .../java/org/apache/myriad/state/NodeTask.java | 119 ++++ .../org/apache/myriad/state/SchedulerState.java | 549 +++++++++++++++++++ .../myriad/state/utils/ByteBufferSupport.java | 368 +++++++++++++ .../apache/myriad/state/utils/StoreContext.java | 276 ++++++++++ .../myriad/webapp/HttpConnectorProvider.java | 49 ++ .../myriad/webapp/MyriadServletModule.java | 44 ++ .../apache/myriad/webapp/MyriadWebServer.java | 72 +++ .../apache/myriad/webapp/WebAppGuiceModule.java | 34 ++ .../src/main/resources/yarn-site-default.xml | 4 +- .../test/java/com/ebay/myriad/MesosModule.java | 83 --- .../java/com/ebay/myriad/MultiBindingsTest.java | 71 --- .../com/ebay/myriad/MultiBindingsUsage.java | 38 -- .../java/com/ebay/myriad/MyriadTestModule.java | 103 ---- .../MyriadBadConfigurationExceptionTest.java | 48 -- .../configuration/MyriadConfigurationTest.java | 66 --- .../myriad/scheduler/SchedulerUtilsSpec.groovy | 90 --- .../myriad/scheduler/TMSTaskFactoryImpl.java | 72 --- .../myriad/scheduler/TestMyriadScheduler.java | 115 ---- .../scheduler/TestServiceCommandLine.java | 83 --- .../ebay/myriad/scheduler/TestTaskUtils.java | 100 ---- .../constraints/LikeConstraintSpec.groovy | 93 ---- .../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 170 ------ .../scheduler/fgs/NMHeartBeatHandlerSpec.groovy | 114 ---- .../fgs/YarnNodeCapacityManagerSpec.groovy | 137 ----- .../java/org/apache/myriad/MesosModule.java | 80 +++ .../org/apache/myriad/MultiBindingsTest.java | 71 +++ .../org/apache/myriad/MultiBindingsUsage.java | 38 ++ .../org/apache/myriad/MyriadTestModule.java | 100 ++++ .../MyriadBadConfigurationExceptionTest.java | 48 ++ .../configuration/MyriadConfigurationTest.java | 66 +++ .../myriad/scheduler/SchedulerUtilsSpec.groovy | 90 +++ .../myriad/scheduler/TMSTaskFactoryImpl.java | 71 +++ .../myriad/scheduler/TestMyriadScheduler.java | 114 ++++ .../scheduler/TestServiceCommandLine.java | 83 +++ .../apache/myriad/scheduler/TestTaskUtils.java | 100 ++++ .../constraints/LikeConstraintSpec.groovy | 93 ++++ .../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 170 ++++++ .../scheduler/fgs/NMHeartBeatHandlerSpec.groovy | 114 ++++ .../fgs/YarnNodeCapacityManagerSpec.groovy | 135 +++++ 273 files changed, 11878 insertions(+), 11960 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index b2694b4..74ad8b1 100644 --- a/build.gradle +++ b/build.gradle @@ -46,9 +46,9 @@ subprojects { [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' - mainClassName = "com.ebay.myriad.Main" + mainClassName = "org.apache.myriad.Main" - group = "com.ebay.myriad" + group = "org.apache.myriad" version = "0.0.1" ext { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/config/findbugs/excludeFilter.xml ---------------------------------------------------------------------- diff --git a/config/findbugs/excludeFilter.xml b/config/findbugs/excludeFilter.xml index c50cf5c..ba32609 100644 --- a/config/findbugs/excludeFilter.xml +++ b/config/findbugs/excludeFilter.xml @@ -15,7 +15,7 @@ <Match> <!-- generated packages--> - <Package name="com.ebay.myriad.scheduler.event"/> + <Package name="org.apache.myriad.scheduler.event"/> </Match> <Match> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/docs/myriad-dev.md ---------------------------------------------------------------------- diff --git a/docs/myriad-dev.md b/docs/myriad-dev.md index 29c3071..4a11682 100644 --- a/docs/myriad-dev.md +++ b/docs/myriad-dev.md @@ -121,8 +121,8 @@ export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so <!-- Configure Myriad Scheduler here --> <property> <name>yarn.resourcemanager.scheduler.class</name> - <value>com.ebay.myriad.scheduler.yarn.MyriadFairScheduler</value> - <description>One can configure other schedulers as well from following list: com.ebay.myriad.scheduler.yarn.MyriadCapacityScheduler, com.ebay.myriad.scheduler.yarn.MyriadFifoScheduler</description> + <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value> + <description>One can configure other schedulers as well from following list: org.apache.myriad.scheduler.yarn.MyriadCapacityScheduler, org.apache.myriad.scheduler.yarn.MyriadFifoScheduler</description> </property> <property> <description>A comma separated list of services where service name should only contain a-zA-Z0-9_ and can not start with numbers</description> @@ -133,7 +133,7 @@ export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so </property> <property> <name>yarn.nodemanager.aux-services.myriad_executor.class</name> - <value>com.ebay.myriad.executor.MyriadExecutorAuxService</value> + <value>org.apache.myriad.executor.MyriadExecutorAuxService</value> </property> ``` http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/docs/samples/yarn-site.xml ---------------------------------------------------------------------- diff --git a/docs/samples/yarn-site.xml b/docs/samples/yarn-site.xml index 2cf4917..2c36c34 100644 --- a/docs/samples/yarn-site.xml +++ b/docs/samples/yarn-site.xml @@ -32,7 +32,7 @@ <!-- Configure Myriad Scheduler here --> <property> <name>yarn.resourcemanager.scheduler.class</name> - <value>com.ebay.myriad.scheduler.yarn.MyriadFairScheduler</value> + <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value> </property> <!-- Cgroups specific configuration --> @@ -66,4 +66,4 @@ <name>yarn.nodemanager.linux-container-executor.path</name> <value>${yarn.home}/bin/container-executor</value> </property> -</configuration> \ No newline at end of file +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/docs/vagrant.md ---------------------------------------------------------------------- diff --git a/docs/vagrant.md b/docs/vagrant.md index 50a96b3..17be2d2 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -87,8 +87,8 @@ To configure YARN to use Myriad, please update ```$YARN_HOME/etc/hadoop/yarn-sit <!-- Configure Myriad Scheduler here --> <property> <name>yarn.resourcemanager.scheduler.class</name> - <value>com.ebay.myriad.scheduler.yarn.MyriadFairScheduler</value> - <description>One can configure other schedulers as well from following list: com.ebay.myriad.scheduler.yarn.MyriadCapacityScheduler, com.ebay.myriad.scheduler.yarn.MyriadFifoScheduler</description> + <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value> + <description>One can configure other schedulers as well from following list: org.apache.myriad.scheduler.yarn.MyriadCapacityScheduler, org.apache.myriad.scheduler.yarn.MyriadFifoScheduler</description> </property> ``` http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/config/findbugs/excludeFilter.xml ---------------------------------------------------------------------- diff --git a/myriad-commons/config/findbugs/excludeFilter.xml b/myriad-commons/config/findbugs/excludeFilter.xml index c50cf5c..ba32609 100644 --- a/myriad-commons/config/findbugs/excludeFilter.xml +++ b/myriad-commons/config/findbugs/excludeFilter.xml @@ -15,7 +15,7 @@ <Match> <!-- generated packages--> - <Package name="com.ebay.myriad.scheduler.event"/> + <Package name="org.apache.myriad.scheduler.event"/> </Match> <Match> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/com/ebay/myriad/executor/ContainerTaskStatusRequest.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/com/ebay/myriad/executor/ContainerTaskStatusRequest.java b/myriad-commons/src/main/java/com/ebay/myriad/executor/ContainerTaskStatusRequest.java deleted file mode 100644 index a93e3a8..0000000 --- a/myriad-commons/src/main/java/com/ebay/myriad/executor/ContainerTaskStatusRequest.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.executor; - -/** - * Sent as a "framework message" to the executor. The executor sends a "status report" for - * the mesos task id (placeholder task). - */ -public class ContainerTaskStatusRequest { - public static final String YARN_CONTAINER_TASK_ID_PREFIX = "yarn_"; - private String mesosTaskId; // YARN_CONTAINER_TASK_ID_PREFIX + <container_id> - private String state; // Protos.TaskState.name() - - public String getMesosTaskId() { - return mesosTaskId; - } - - public void setMesosTaskId(String mesosTaskId) { - this.mesosTaskId = mesosTaskId; - } - - public String getState() { - return state; - } - - public void setState(String state) { - this.state = state; - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/com/ebay/myriad/executor/MyriadExecutorDefaults.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/com/ebay/myriad/executor/MyriadExecutorDefaults.java b/myriad-commons/src/main/java/com/ebay/myriad/executor/MyriadExecutorDefaults.java deleted file mode 100644 index 32925e5..0000000 --- a/myriad-commons/src/main/java/com/ebay/myriad/executor/MyriadExecutorDefaults.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.executor; - -/** - * Myriad's Executor Defaults - */ -public class MyriadExecutorDefaults { - public static final String ENV_YARN_NODEMANAGER_OPTS = "YARN_NODEMANAGER_OPTS"; - - /** - * YARN container executor class. - */ - public static final String KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS = "yarn.nodemanager.container-executor.class"; - - public static final String VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"; - - public static final String DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS = "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"; - - /** - * YARN class to help handle LCE resources - */ - public static final String KEY_YARN_NM_LCE_RH_CLASS = "yarn.nodemanager.linux-container-executor.resources-handler.class"; - - public static final String VAL_YARN_NM_LCE_RH_CLASS = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; - - public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierarchy"; - - public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT = "yarn.nodemanager.linux-container-executor.cgroups.mount"; - - public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; - - public static final String KEY_YARN_NM_LCE_GROUP = "yarn.nodemanager.linux-container-executor.group"; - - public static final String KEY_YARN_NM_LCE_PATH = "yarn.nodemanager.linux-container-executor.path"; - - public static final String KEY_YARN_HOME = "yarn.home"; - - public static final String KEY_NM_RESOURCE_CPU_VCORES = "nodemanager.resource.cpu-vcores"; - - public static final String KEY_NM_RESOURCE_MEM_MB = "nodemanager.resource.memory-mb"; - - /** - * Allot 10% more memory to account for JVM overhead. - */ - public static final double JVM_OVERHEAD = 0.1; - - /** - * Default -Xmx for executor JVM. - */ - - public static final double DEFAULT_JVM_MAX_MEMORY_MB = 256; - /** - * Default cpus for executor JVM. - */ - public static final double DEFAULT_CPUS = 0.2; - - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/com/ebay/myriad/executor/NMTaskConfig.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/com/ebay/myriad/executor/NMTaskConfig.java b/myriad-commons/src/main/java/com/ebay/myriad/executor/NMTaskConfig.java deleted file mode 100644 index 92626b3..0000000 --- a/myriad-commons/src/main/java/com/ebay/myriad/executor/NMTaskConfig.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.executor; - -import java.util.Map; - -/** - * Node Manger Task Configuraiton - */ -public class NMTaskConfig { - private String yarnHome; - private Long advertisableCpus; - private Long advertisableMem; - private String jvmOpts; - private Boolean cgroups; - private Long rpcPort; - private Long localizerPort; - private Long webAppHttpPort; - private Long shufflePort; - - private Map<String, String> yarnEnvironment; - - public String getYarnHome() { - return yarnHome; - } - - public void setYarnHome(String yarnHome) { - this.yarnHome = yarnHome; - } - - public Long getAdvertisableCpus() { - return advertisableCpus; - } - - public void setAdvertisableCpus(Long advertisableCpus) { - this.advertisableCpus = advertisableCpus; - } - - public Long getAdvertisableMem() { - return advertisableMem; - } - - public void setAdvertisableMem(Long advertisableMem) { - this.advertisableMem = advertisableMem; - } - - public String getJvmOpts() { - return jvmOpts; - } - - public void setJvmOpts(String jvmOpts) { - this.jvmOpts = jvmOpts; - } - - public Boolean getCgroups() { - return cgroups; - } - - public void setCgroups(Boolean cgroups) { - this.cgroups = cgroups; - } - - public Map<String, String> getYarnEnvironment() { - return yarnEnvironment; - } - - public void setYarnEnvironment(Map<String, String> yarnEnvironment) { - this.yarnEnvironment = yarnEnvironment; - } - - public Long getRpcPort() { - return rpcPort; - } - - public void setRpcPort(long port) { - rpcPort = port; - } - - public Long gettWebAppHttpPort() { - return webAppHttpPort; - } - - public void setWebAppHttpPort(Long port) { - webAppHttpPort = port; - } - - public Long getLocalizerPort() { - return localizerPort; - } - - public void setLocalizerPort(Long localizerPort) { - this.localizerPort = localizerPort; - } - - public Long getShufflePort() { - return shufflePort; - } - - public void setShufflePort(Long shufflePort) { - this.shufflePort = shufflePort; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/org/apache/myriad/executor/ContainerTaskStatusRequest.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/org/apache/myriad/executor/ContainerTaskStatusRequest.java b/myriad-commons/src/main/java/org/apache/myriad/executor/ContainerTaskStatusRequest.java new file mode 100644 index 0000000..4205622 --- /dev/null +++ b/myriad-commons/src/main/java/org/apache/myriad/executor/ContainerTaskStatusRequest.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.executor; + +/** + * Sent as a "framework message" to the executor. The executor sends a "status report" for + * the mesos task id (placeholder task). + */ +public class ContainerTaskStatusRequest { + public static final String YARN_CONTAINER_TASK_ID_PREFIX = "yarn_"; + private String mesosTaskId; // YARN_CONTAINER_TASK_ID_PREFIX + <container_id> + private String state; // Protos.TaskState.name() + + public String getMesosTaskId() { + return mesosTaskId; + } + + public void setMesosTaskId(String mesosTaskId) { + this.mesosTaskId = mesosTaskId; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java new file mode 100644 index 0000000..eee4a60 --- /dev/null +++ b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.executor; + +/** + * Myriad's Executor Defaults + */ +public class MyriadExecutorDefaults { + public static final String ENV_YARN_NODEMANAGER_OPTS = "YARN_NODEMANAGER_OPTS"; + + /** + * YARN container executor class. + */ + public static final String KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS = "yarn.nodemanager.container-executor.class"; + + public static final String VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"; + + public static final String DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS = "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"; + + /** + * YARN class to help handle LCE resources + */ + public static final String KEY_YARN_NM_LCE_RH_CLASS = "yarn.nodemanager.linux-container-executor.resources-handler.class"; + + public static final String VAL_YARN_NM_LCE_RH_CLASS = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; + + public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierarchy"; + + public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT = "yarn.nodemanager.linux-container-executor.cgroups.mount"; + + public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path"; + + public static final String KEY_YARN_NM_LCE_GROUP = "yarn.nodemanager.linux-container-executor.group"; + + public static final String KEY_YARN_NM_LCE_PATH = "yarn.nodemanager.linux-container-executor.path"; + + public static final String KEY_YARN_HOME = "yarn.home"; + + public static final String KEY_NM_RESOURCE_CPU_VCORES = "nodemanager.resource.cpu-vcores"; + + public static final String KEY_NM_RESOURCE_MEM_MB = "nodemanager.resource.memory-mb"; + + /** + * Allot 10% more memory to account for JVM overhead. + */ + public static final double JVM_OVERHEAD = 0.1; + + /** + * Default -Xmx for executor JVM. + */ + + public static final double DEFAULT_JVM_MAX_MEMORY_MB = 256; + /** + * Default cpus for executor JVM. + */ + public static final double DEFAULT_CPUS = 0.2; + + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java ---------------------------------------------------------------------- diff --git a/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java b/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java new file mode 100644 index 0000000..21d2420 --- /dev/null +++ b/myriad-commons/src/main/java/org/apache/myriad/executor/NMTaskConfig.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.executor; + +import java.util.Map; + +/** + * Node Manger Task Configuraiton + */ +public class NMTaskConfig { + private String yarnHome; + private Long advertisableCpus; + private Long advertisableMem; + private String jvmOpts; + private Boolean cgroups; + private Long rpcPort; + private Long localizerPort; + private Long webAppHttpPort; + private Long shufflePort; + + private Map<String, String> yarnEnvironment; + + public String getYarnHome() { + return yarnHome; + } + + public void setYarnHome(String yarnHome) { + this.yarnHome = yarnHome; + } + + public Long getAdvertisableCpus() { + return advertisableCpus; + } + + public void setAdvertisableCpus(Long advertisableCpus) { + this.advertisableCpus = advertisableCpus; + } + + public Long getAdvertisableMem() { + return advertisableMem; + } + + public void setAdvertisableMem(Long advertisableMem) { + this.advertisableMem = advertisableMem; + } + + public String getJvmOpts() { + return jvmOpts; + } + + public void setJvmOpts(String jvmOpts) { + this.jvmOpts = jvmOpts; + } + + public Boolean getCgroups() { + return cgroups; + } + + public void setCgroups(Boolean cgroups) { + this.cgroups = cgroups; + } + + public Map<String, String> getYarnEnvironment() { + return yarnEnvironment; + } + + public void setYarnEnvironment(Map<String, String> yarnEnvironment) { + this.yarnEnvironment = yarnEnvironment; + } + + public Long getRpcPort() { + return rpcPort; + } + + public void setRpcPort(long port) { + rpcPort = port; + } + + public Long gettWebAppHttpPort() { + return webAppHttpPort; + } + + public void setWebAppHttpPort(Long port) { + webAppHttpPort = port; + } + + public Long getLocalizerPort() { + return localizerPort; + } + + public void setLocalizerPort(Long localizerPort) { + this.localizerPort = localizerPort; + } + + public Long getShufflePort() { + return shufflePort; + } + + public void setShufflePort(Long shufflePort) { + this.shufflePort = shufflePort; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/build.gradle ---------------------------------------------------------------------- diff --git a/myriad-executor/build.gradle b/myriad-executor/build.gradle index 65879df..15529c9 100644 --- a/myriad-executor/build.gradle +++ b/myriad-executor/build.gradle @@ -38,7 +38,7 @@ task capsule(type: Jar, dependsOn: jar) { manifest { attributes( 'Main-Class': 'Capsule', - 'Application-Class': 'com.ebay.myriad.executor.MyriadExecutor', + 'Application-Class': 'org.apache.myriad.executor.MyriadExecutor', 'Min-Java-Version': '1.7.0', 'JVM-Args': run.jvmArgs.join(' '), 'System-Properties': (run.systemProperties).collect { k, v -> "$k=$v" }.join(' ') http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/config/findbugs/excludeFilter.xml ---------------------------------------------------------------------- diff --git a/myriad-executor/config/findbugs/excludeFilter.xml b/myriad-executor/config/findbugs/excludeFilter.xml index c50cf5c..ba32609 100644 --- a/myriad-executor/config/findbugs/excludeFilter.xml +++ b/myriad-executor/config/findbugs/excludeFilter.xml @@ -15,7 +15,7 @@ <Match> <!-- generated packages--> - <Package name="com.ebay.myriad.scheduler.event"/> + <Package name="org.apache.myriad.scheduler.event"/> </Match> <Match> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutor.java ---------------------------------------------------------------------- diff --git a/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutor.java b/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutor.java deleted file mode 100644 index 37926d4..0000000 --- a/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutor.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad.executor; - -import org.apache.mesos.Executor; -import org.apache.mesos.ExecutorDriver; -import org.apache.mesos.Protos; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.FrameworkInfo; -import org.apache.mesos.Protos.SlaveInfo; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.apache.mesos.Protos.TaskState; -import org.apache.mesos.Protos.TaskStatus; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.Charset; -import java.util.Set; - -/** - * Myriad's Executor - */ -public class MyriadExecutor implements Executor { - - private static final Logger LOGGER = LoggerFactory.getLogger(MyriadExecutor.class); - - private Set<String> containerIds; - - public MyriadExecutor(Set<String> containerTaskIds) { - this.containerIds = containerTaskIds; - } - - @Override - public void registered(ExecutorDriver driver, ExecutorInfo executorInfo, FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) { - LOGGER.debug("Registered ", executorInfo, " for framework ", frameworkInfo, " on mesos slave ", slaveInfo); - } - - @Override - public void reregistered(ExecutorDriver driver, SlaveInfo slaveInfo) { - LOGGER.debug("ReRegistered"); - } - - @Override - public void disconnected(ExecutorDriver driver) { - LOGGER.info("Disconnected"); - } - - @Override - public void launchTask(final ExecutorDriver driver, final TaskInfo task) { - LOGGER.debug("launchTask received for taskId: " + task.getTaskId()); - TaskStatus status = TaskStatus.newBuilder().setTaskId(task.getTaskId()).setState(TaskState.TASK_RUNNING).build(); - driver.sendStatusUpdate(status); - } - - @Override - public void killTask(ExecutorDriver driver, TaskID taskId) { - LOGGER.debug("killTask received for taskId: " + taskId.getValue()); - TaskStatus status; - - if (!taskId.toString().contains(MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX)) { - // Inform mesos of killing all tasks corresponding to yarn containers that are - // currently running - synchronized (containerIds) { - for (String containerId : containerIds) { - Protos.TaskID containerTaskId = Protos.TaskID.newBuilder().setValue(MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX + containerId).build(); - status = TaskStatus.newBuilder().setTaskId(containerTaskId).setState(TaskState.TASK_KILLED).build(); - driver.sendStatusUpdate(status); - } - } - - // Now kill the node manager task - status = TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_KILLED).build(); - driver.sendStatusUpdate(status); - LOGGER.info("NodeManager shutdown after receiving" + - " KillTask for taskId " + taskId.getValue()); - Runtime.getRuntime().exit(0); - - } else { - LOGGER.debug("Cannot delete tasks corresponding to yarn container " + taskId); - } - } - - @Override - public void frameworkMessage(ExecutorDriver driver, byte[] data) { - LOGGER.info("Framework message received: ", new String(data, Charset.defaultCharset())); - } - - @Override - public void shutdown(ExecutorDriver driver) { - LOGGER.debug("Shutdown"); - } - - @Override - public void error(ExecutorDriver driver, String message) { - LOGGER.error("Error message: " + message); - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutorAuxService.java ---------------------------------------------------------------------- diff --git a/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutorAuxService.java b/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutorAuxService.java deleted file mode 100644 index 86ea60e..0000000 --- a/myriad-executor/src/main/java/com/ebay/myriad/executor/MyriadExecutorAuxService.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.ebay.myriad.executor; - -import java.nio.ByteBuffer; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; -import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; -import org.apache.hadoop.yarn.server.api.AuxiliaryService; -import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; -import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; - -import org.apache.mesos.MesosExecutorDriver; -import org.apache.mesos.Protos.Status; -import org.apache.mesos.Protos.TaskState; -import org.apache.mesos.Protos.TaskStatus; -import org.apache.mesos.Protos; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Auxillary service wrapper for MyriadExecutor - */ -public class MyriadExecutorAuxService extends AuxiliaryService { - - private static final Logger LOGGER = LoggerFactory.getLogger(MyriadExecutor.class); - private static final String SERVICE_NAME = "myriad_service"; - public static final String YARN_CONTAINER_TASK_ID_PREFIX = "yarn_"; - - private MesosExecutorDriver driver; - private Thread myriadExecutorThread; - // Storing container id strings as it is difficult to get access to - // NodeManager's NMContext object from an auxiliary service. - private Set<String> containerIds = new HashSet<>(); - - protected MyriadExecutorAuxService() { - super(SERVICE_NAME); - } - - @Override - protected void serviceStart() throws Exception { - LOGGER.info("Starting MyriadExecutor..."); - - myriadExecutorThread = new Thread(new Runnable() { - public void run() { - driver = new MesosExecutorDriver(new MyriadExecutor(containerIds)); - LOGGER.error("MyriadExecutor exit with status " + Integer.toString(driver.run() == Status.DRIVER_STOPPED ? 0 : 1)); - } - }); - myriadExecutorThread.start(); - } - - @Override - public void initializeApplication(ApplicationInitializationContext initAppContext) { - LOGGER.debug("initializeApplication"); - } - - @Override - public void stopApplication(ApplicationTerminationContext stopAppContext) { - LOGGER.debug("stopApplication"); - } - - @Override - public ByteBuffer getMetaData() { - LOGGER.debug("getMetaData"); - return null; - } - - @Override - public void initializeContainer(ContainerInitializationContext initContainerContext) { - ContainerId containerId = initContainerContext.getContainerId(); - synchronized (containerIds) { - containerIds.add(containerId.toString()); - } - sendStatus(containerId, TaskState.TASK_RUNNING); - } - - @Override - public void stopContainer(ContainerTerminationContext stopContainerContext) { - ContainerId containerId = stopContainerContext.getContainerId(); - synchronized (containerIds) { - containerIds.remove(containerId.toString()); - } - sendStatus(stopContainerContext.getContainerId(), TaskState.TASK_FINISHED); - } - - private void sendStatus(ContainerId containerId, TaskState taskState) { - Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(YARN_CONTAINER_TASK_ID_PREFIX + containerId.toString()).build(); - - TaskStatus status = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); - driver.sendStatusUpdate(status); - LOGGER.debug("Sent status " + taskState + " for taskId " + taskId); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutor.java ---------------------------------------------------------------------- diff --git a/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutor.java b/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutor.java new file mode 100644 index 0000000..cc9b9af --- /dev/null +++ b/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutor.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.executor; + +import org.apache.mesos.Executor; +import org.apache.mesos.ExecutorDriver; +import org.apache.mesos.Protos; +import org.apache.mesos.Protos.ExecutorInfo; +import org.apache.mesos.Protos.FrameworkInfo; +import org.apache.mesos.Protos.SlaveInfo; +import org.apache.mesos.Protos.TaskID; +import org.apache.mesos.Protos.TaskInfo; +import org.apache.mesos.Protos.TaskState; +import org.apache.mesos.Protos.TaskStatus; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.Charset; +import java.util.Set; + +/** + * Myriad's Executor + */ +public class MyriadExecutor implements Executor { + + private static final Logger LOGGER = LoggerFactory.getLogger(MyriadExecutor.class); + + private Set<String> containerIds; + + public MyriadExecutor(Set<String> containerTaskIds) { + this.containerIds = containerTaskIds; + } + + @Override + public void registered(ExecutorDriver driver, ExecutorInfo executorInfo, FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) { + LOGGER.debug("Registered ", executorInfo, " for framework ", frameworkInfo, " on mesos slave ", slaveInfo); + } + + @Override + public void reregistered(ExecutorDriver driver, SlaveInfo slaveInfo) { + LOGGER.debug("ReRegistered"); + } + + @Override + public void disconnected(ExecutorDriver driver) { + LOGGER.info("Disconnected"); + } + + @Override + public void launchTask(final ExecutorDriver driver, final TaskInfo task) { + LOGGER.debug("launchTask received for taskId: " + task.getTaskId()); + TaskStatus status = TaskStatus.newBuilder().setTaskId(task.getTaskId()).setState(TaskState.TASK_RUNNING).build(); + driver.sendStatusUpdate(status); + } + + @Override + public void killTask(ExecutorDriver driver, TaskID taskId) { + LOGGER.debug("killTask received for taskId: " + taskId.getValue()); + TaskStatus status; + + if (!taskId.toString().contains(MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX)) { + // Inform mesos of killing all tasks corresponding to yarn containers that are + // currently running + synchronized (containerIds) { + for (String containerId : containerIds) { + Protos.TaskID containerTaskId = Protos.TaskID.newBuilder().setValue(MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX + containerId).build(); + status = TaskStatus.newBuilder().setTaskId(containerTaskId).setState(TaskState.TASK_KILLED).build(); + driver.sendStatusUpdate(status); + } + } + + // Now kill the node manager task + status = TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_KILLED).build(); + driver.sendStatusUpdate(status); + LOGGER.info("NodeManager shutdown after receiving" + + " KillTask for taskId " + taskId.getValue()); + Runtime.getRuntime().exit(0); + + } else { + LOGGER.debug("Cannot delete tasks corresponding to yarn container " + taskId); + } + } + + @Override + public void frameworkMessage(ExecutorDriver driver, byte[] data) { + LOGGER.info("Framework message received: ", new String(data, Charset.defaultCharset())); + } + + @Override + public void shutdown(ExecutorDriver driver) { + LOGGER.debug("Shutdown"); + } + + @Override + public void error(ExecutorDriver driver, String message) { + LOGGER.error("Error message: " + message); + } +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutorAuxService.java ---------------------------------------------------------------------- diff --git a/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutorAuxService.java b/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutorAuxService.java new file mode 100644 index 0000000..322f124 --- /dev/null +++ b/myriad-executor/src/main/java/org/apache/myriad/executor/MyriadExecutorAuxService.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.myriad.executor; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext; +import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; +import org.apache.hadoop.yarn.server.api.AuxiliaryService; +import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; +import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; + +import org.apache.mesos.MesosExecutorDriver; +import org.apache.mesos.Protos.Status; +import org.apache.mesos.Protos.TaskState; +import org.apache.mesos.Protos.TaskStatus; +import org.apache.mesos.Protos; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Auxillary service wrapper for MyriadExecutor + */ +public class MyriadExecutorAuxService extends AuxiliaryService { + + private static final Logger LOGGER = LoggerFactory.getLogger(MyriadExecutor.class); + private static final String SERVICE_NAME = "myriad_service"; + public static final String YARN_CONTAINER_TASK_ID_PREFIX = "yarn_"; + + private MesosExecutorDriver driver; + private Thread myriadExecutorThread; + // Storing container id strings as it is difficult to get access to + // NodeManager's NMContext object from an auxiliary service. + private Set<String> containerIds = new HashSet<>(); + + protected MyriadExecutorAuxService() { + super(SERVICE_NAME); + } + + @Override + protected void serviceStart() throws Exception { + LOGGER.info("Starting MyriadExecutor..."); + + myriadExecutorThread = new Thread(new Runnable() { + public void run() { + driver = new MesosExecutorDriver(new MyriadExecutor(containerIds)); + LOGGER.error("MyriadExecutor exit with status " + Integer.toString(driver.run() == Status.DRIVER_STOPPED ? 0 : 1)); + } + }); + myriadExecutorThread.start(); + } + + @Override + public void initializeApplication(ApplicationInitializationContext initAppContext) { + LOGGER.debug("initializeApplication"); + } + + @Override + public void stopApplication(ApplicationTerminationContext stopAppContext) { + LOGGER.debug("stopApplication"); + } + + @Override + public ByteBuffer getMetaData() { + LOGGER.debug("getMetaData"); + return null; + } + + @Override + public void initializeContainer(ContainerInitializationContext initContainerContext) { + ContainerId containerId = initContainerContext.getContainerId(); + synchronized (containerIds) { + containerIds.add(containerId.toString()); + } + sendStatus(containerId, TaskState.TASK_RUNNING); + } + + @Override + public void stopContainer(ContainerTerminationContext stopContainerContext) { + ContainerId containerId = stopContainerContext.getContainerId(); + synchronized (containerIds) { + containerIds.remove(containerId.toString()); + } + sendStatus(stopContainerContext.getContainerId(), TaskState.TASK_FINISHED); + } + + private void sendStatus(ContainerId containerId, TaskState taskState) { + Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(YARN_CONTAINER_TASK_ID_PREFIX + containerId.toString()).build(); + + TaskStatus status = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build(); + driver.sendStatusUpdate(status); + LOGGER.debug("Sent status " + taskState + " for taskId " + taskId); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/config/findbugs/excludeFilter.xml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/config/findbugs/excludeFilter.xml b/myriad-scheduler/config/findbugs/excludeFilter.xml index c50cf5c..ba32609 100644 --- a/myriad-scheduler/config/findbugs/excludeFilter.xml +++ b/myriad-scheduler/config/findbugs/excludeFilter.xml @@ -15,7 +15,7 @@ <Match> <!-- generated packages--> - <Package name="com.ebay.myriad.scheduler.event"/> + <Package name="org.apache.myriad.scheduler.event"/> </Match> <Match> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/main/java/com/ebay/myriad/DisruptorManager.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/DisruptorManager.java b/myriad-scheduler/src/main/java/com/ebay/myriad/DisruptorManager.java deleted file mode 100644 index e3fb399..0000000 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/DisruptorManager.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad; - -import com.ebay.myriad.scheduler.event.*; -import com.ebay.myriad.scheduler.event.handlers.DisconnectedEventHandler; -import com.ebay.myriad.scheduler.event.handlers.ErrorEventHandler; -import com.ebay.myriad.scheduler.event.handlers.ExecutorLostEventHandler; -import com.ebay.myriad.scheduler.event.handlers.FrameworkMessageEventHandler; -import com.ebay.myriad.scheduler.event.handlers.OfferRescindedEventHandler; -import com.ebay.myriad.scheduler.event.handlers.ReRegisteredEventHandler; -import com.ebay.myriad.scheduler.event.handlers.RegisteredEventHandler; -import com.ebay.myriad.scheduler.event.handlers.ResourceOffersEventHandler; -import com.ebay.myriad.scheduler.event.handlers.SlaveLostEventHandler; -import com.ebay.myriad.scheduler.event.handlers.StatusUpdateEventHandler; -import com.google.inject.Injector; -import com.lmax.disruptor.dsl.Disruptor; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Disruptor class is an event bus used in high speed financial systems. http://martinfowler.com/articles/lmax.html - * Here it is used to abstract incoming events. - */ -public class DisruptorManager { - private ExecutorService disruptorExecutors; - - private static final int DEFAULT_SMALL_RINGBUFFER_SIZE = 64; - private static final int DEFAULT_LARGE_RINGBUFFER_SIZE = 1024; - - private Disruptor<RegisteredEvent> registeredEventDisruptor; - private Disruptor<ReRegisteredEvent> reRegisteredEventDisruptor; - private Disruptor<ResourceOffersEvent> resourceOffersEventDisruptor; - private Disruptor<OfferRescindedEvent> offerRescindedEventDisruptor; - private Disruptor<StatusUpdateEvent> statusUpdateEventDisruptor; - private Disruptor<FrameworkMessageEvent> frameworkMessageEventDisruptor; - private Disruptor<DisconnectedEvent> disconnectedEventDisruptor; - private Disruptor<SlaveLostEvent> slaveLostEventDisruptor; - private Disruptor<ExecutorLostEvent> executorLostEventDisruptor; - private Disruptor<ErrorEvent> errorEventDisruptor; - - @SuppressWarnings("unchecked") - public void init(Injector injector) { - this.disruptorExecutors = Executors.newCachedThreadPool(); - - // todo: (kensipe) need to make ringsize configurable (overriding the defaults) - - - this.registeredEventDisruptor = new Disruptor<>(new RegisteredEventFactory(), DEFAULT_SMALL_RINGBUFFER_SIZE, disruptorExecutors); - this.registeredEventDisruptor.handleEventsWith(injector.getInstance(RegisteredEventHandler.class)); - this.registeredEventDisruptor.start(); - - this.reRegisteredEventDisruptor = new Disruptor<>(new ReRegisteredEventFactory(), DEFAULT_SMALL_RINGBUFFER_SIZE, disruptorExecutors); - this.reRegisteredEventDisruptor.handleEventsWith(injector.getInstance(ReRegisteredEventHandler.class)); - this.reRegisteredEventDisruptor.start(); - - - this.resourceOffersEventDisruptor = new Disruptor<>(new ResourceOffersEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.resourceOffersEventDisruptor.handleEventsWith(injector.getInstance(ResourceOffersEventHandler.class)); - this.resourceOffersEventDisruptor.start(); - - this.offerRescindedEventDisruptor = new Disruptor<>(new OfferRescindedEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.offerRescindedEventDisruptor.handleEventsWith(injector.getInstance(OfferRescindedEventHandler.class)); - this.offerRescindedEventDisruptor.start(); - - this.statusUpdateEventDisruptor = new Disruptor<>(new StatusUpdateEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.statusUpdateEventDisruptor.handleEventsWith(injector.getInstance(StatusUpdateEventHandler.class)); - this.statusUpdateEventDisruptor.start(); - - this.frameworkMessageEventDisruptor = new Disruptor<>(new FrameworkMessageEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.frameworkMessageEventDisruptor.handleEventsWith(injector.getInstance(FrameworkMessageEventHandler.class)); - this.frameworkMessageEventDisruptor.start(); - - this.disconnectedEventDisruptor = new Disruptor<>(new DisconnectedEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.disconnectedEventDisruptor.handleEventsWith(injector.getInstance(DisconnectedEventHandler.class)); - this.disconnectedEventDisruptor.start(); - - this.slaveLostEventDisruptor = new Disruptor<>(new SlaveLostEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.slaveLostEventDisruptor.handleEventsWith(injector.getInstance(SlaveLostEventHandler.class)); - this.slaveLostEventDisruptor.start(); - - this.executorLostEventDisruptor = new Disruptor<>(new ExecutorLostEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.executorLostEventDisruptor.handleEventsWith(injector.getInstance(ExecutorLostEventHandler.class)); - this.executorLostEventDisruptor.start(); - - this.errorEventDisruptor = new Disruptor<>(new ErrorEventFactory(), DEFAULT_LARGE_RINGBUFFER_SIZE, disruptorExecutors); - this.errorEventDisruptor.handleEventsWith(injector.getInstance(ErrorEventHandler.class)); - this.errorEventDisruptor.start(); - } - - public Disruptor<RegisteredEvent> getRegisteredEventDisruptor() { - return registeredEventDisruptor; - } - - public Disruptor<ReRegisteredEvent> getReRegisteredEventDisruptor() { - return reRegisteredEventDisruptor; - } - - public Disruptor<ResourceOffersEvent> getResourceOffersEventDisruptor() { - return resourceOffersEventDisruptor; - } - - public Disruptor<OfferRescindedEvent> getOfferRescindedEventDisruptor() { - return offerRescindedEventDisruptor; - } - - public Disruptor<StatusUpdateEvent> getStatusUpdateEventDisruptor() { - return statusUpdateEventDisruptor; - } - - public Disruptor<FrameworkMessageEvent> getFrameworkMessageEventDisruptor() { - return frameworkMessageEventDisruptor; - } - - public Disruptor<DisconnectedEvent> getDisconnectedEventDisruptor() { - return disconnectedEventDisruptor; - } - - public Disruptor<SlaveLostEvent> getSlaveLostEventDisruptor() { - return slaveLostEventDisruptor; - } - - public Disruptor<ExecutorLostEvent> getExecutorLostEventDisruptor() { - return executorLostEventDisruptor; - } - - public Disruptor<ErrorEvent> getErrorEventDisruptor() { - return errorEventDisruptor; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java b/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java deleted file mode 100644 index 8865640..0000000 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java +++ /dev/null @@ -1,299 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad; - -import com.codahale.metrics.JmxReporter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.health.HealthCheckRegistry; -import com.ebay.myriad.configuration.NodeManagerConfiguration; -import com.ebay.myriad.configuration.ServiceConfiguration; -import com.ebay.myriad.configuration.MyriadBadConfigurationException; -import com.ebay.myriad.configuration.MyriadConfiguration; -import com.ebay.myriad.health.MesosDriverHealthCheck; -import com.ebay.myriad.health.MesosMasterHealthCheck; -import com.ebay.myriad.health.ZookeeperHealthCheck; -import com.ebay.myriad.scheduler.ExtendedResourceProfile; -import com.ebay.myriad.scheduler.MyriadDriverManager; -import com.ebay.myriad.scheduler.MyriadOperations; -import com.ebay.myriad.scheduler.NMProfile; -import com.ebay.myriad.scheduler.Rebalancer; -import com.ebay.myriad.scheduler.ServiceProfileManager; -import com.ebay.myriad.scheduler.ServiceResourceProfile; -import com.ebay.myriad.scheduler.ServiceTaskConstraints; -import com.ebay.myriad.scheduler.TaskConstraintsManager; -import com.ebay.myriad.scheduler.TaskFactory; -import com.ebay.myriad.scheduler.TaskTerminator; -import com.ebay.myriad.scheduler.TaskUtils; -import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry; -import com.ebay.myriad.state.NodeTask; -import com.ebay.myriad.state.SchedulerState; -import com.ebay.myriad.webapp.MyriadWebServer; -import com.ebay.myriad.webapp.WebAppGuiceModule; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.inject.Guice; -import com.google.inject.Injector; - -import org.apache.commons.collections.MapUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.HashSet; - -/** - * Main entry point for myriad scheduler - */ -public class Main { - private static final Logger LOGGER = LoggerFactory.getLogger(Main.class); - - private MyriadWebServer webServer; - private ScheduledExecutorService terminatorService; - - private ScheduledExecutorService rebalancerService; - private HealthCheckRegistry healthCheckRegistry; - - private static Injector injector; - - public static void initialize(Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext, InterceptorRegistry registry) throws Exception { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - MyriadConfiguration cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-default.yml"), MyriadConfiguration.class); - - MyriadModule myriadModule = new MyriadModule(cfg, hadoopConf, yarnScheduler, rmContext, registry); - MesosModule mesosModule = new MesosModule(); - injector = Guice.createInjector(myriadModule, mesosModule, new WebAppGuiceModule()); - - new Main().run(cfg); - } - - // TODO (Kannan Rajah) Hack to get injector in unit test. - public static Injector getInjector() { - return injector; - } - - public void run(MyriadConfiguration cfg) throws Exception { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Bindings: " + injector.getAllBindings()); - } - - JmxReporter.forRegistry(new MetricRegistry()).build().start(); - - initWebApp(injector); - initHealthChecks(injector); - initProfiles(injector); - validateNMInstances(injector); - initServiceConfigurations(cfg, injector); - initDisruptors(injector); - - initRebalancerService(cfg, injector); - initTerminatorService(injector); - startMesosDriver(injector); - startNMInstances(injector); - startJavaBasedTaskInstance(injector); - } - - - private void startMesosDriver(Injector injector) { - LOGGER.info("starting mesosDriver.."); - injector.getInstance(MyriadDriverManager.class).startDriver(); - LOGGER.info("started mesosDriver.."); - } - - /** - * Brings up the embedded jetty webserver for serving REST APIs. - * - * @param injector - */ - private void initWebApp(Injector injector) throws Exception { - webServer = injector.getInstance(MyriadWebServer.class); - webServer.start(); - } - - /** - * Initializes health checks. - * - * @param injector - */ - private void initHealthChecks(Injector injector) { - LOGGER.info("Initializing HealthChecks"); - healthCheckRegistry = new HealthCheckRegistry(); - healthCheckRegistry.register(MesosMasterHealthCheck.NAME, injector.getInstance(MesosMasterHealthCheck.class)); - healthCheckRegistry.register(ZookeeperHealthCheck.NAME, injector.getInstance(ZookeeperHealthCheck.class)); - healthCheckRegistry.register(MesosDriverHealthCheck.NAME, injector.getInstance(MesosDriverHealthCheck.class)); - } - - private void initProfiles(Injector injector) { - LOGGER.info("Initializing Profiles"); - ServiceProfileManager profileManager = injector.getInstance(ServiceProfileManager.class); - TaskConstraintsManager taskConstraintsManager = injector.getInstance(TaskConstraintsManager.class); - taskConstraintsManager.addTaskConstraints(NodeManagerConfiguration.NM_TASK_PREFIX, new TaskFactory.NMTaskConstraints()); - Map<String, Map<String, String>> profiles = injector.getInstance(MyriadConfiguration.class).getProfiles(); - TaskUtils taskUtils = injector.getInstance(TaskUtils.class); - if (MapUtils.isNotEmpty(profiles)) { - for (Map.Entry<String, Map<String, String>> profile : profiles.entrySet()) { - Map<String, String> profileResourceMap = profile.getValue(); - if (MapUtils.isNotEmpty(profiles) && profileResourceMap.containsKey("cpu") && profileResourceMap.containsKey("mem")) { - Long cpu = Long.parseLong(profileResourceMap.get("cpu")); - Long mem = Long.parseLong(profileResourceMap.get("mem")); - - ServiceResourceProfile serviceProfile = new ExtendedResourceProfile(new NMProfile(profile.getKey(), cpu, mem), taskUtils.getNodeManagerCpus(), taskUtils.getNodeManagerMemory()); - serviceProfile.setExecutorCpu(taskUtils.getExecutorCpus()); - serviceProfile.setExecutorMemory(taskUtils.getExecutorMemory()); - - profileManager.add(serviceProfile); - } else { - LOGGER.error("Invalid definition for profile: " + profile.getKey()); - } - } - } - } - - private void validateNMInstances(Injector injector) { - LOGGER.info("Validating nmInstances.."); - Map<String, Integer> nmInstances = injector.getInstance(MyriadConfiguration.class).getNmInstances(); - ServiceProfileManager profileManager = injector.getInstance(ServiceProfileManager.class); - - long maxCpu = Long.MIN_VALUE; - long maxMem = Long.MIN_VALUE; - for (Map.Entry<String, Integer> entry : nmInstances.entrySet()) { - String profile = entry.getKey(); - ServiceResourceProfile nodeManager = profileManager.get(profile); - if (nodeManager == null) { - throw new RuntimeException("Invalid profile name '" + profile + "' specified in 'nmInstances'"); - } - if (entry.getValue() > 0) { - if (nodeManager.getCpus() > maxCpu) { // find the profile with largest number of cpus - maxCpu = nodeManager.getCpus().longValue(); - maxMem = nodeManager.getMemory().longValue(); // use the memory from the same profile - } - } - } - if (maxCpu <= 0 || maxMem <= 0) { - throw new RuntimeException("Please configure 'nmInstances' with at least one instance/profile " + "with non-zero cpu/mem resources."); - } - } - - private void startNMInstances(Injector injector) { - Map<String, Integer> nmInstances = injector.getInstance(MyriadConfiguration.class).getNmInstances(); - MyriadOperations myriadOperations = injector.getInstance(MyriadOperations.class); - ServiceProfileManager profileManager = injector.getInstance(ServiceProfileManager.class); - SchedulerState schedulerState = injector.getInstance(SchedulerState.class); - - Set<NodeTask> launchedNMTasks = new HashSet<>(); - launchedNMTasks.addAll(schedulerState.getPendingTasksByType(NodeManagerConfiguration.NM_TASK_PREFIX)); - if (!launchedNMTasks.isEmpty()) { - LOGGER.info("{} NM(s) in pending state. Not launching additional NMs", launchedNMTasks.size()); - return; - } - - launchedNMTasks.addAll(schedulerState.getStagingTasksByType(NodeManagerConfiguration.NM_TASK_PREFIX)); - if (!launchedNMTasks.isEmpty()) { - LOGGER.info("{} NM(s) in staging state. Not launching additional NMs", launchedNMTasks.size()); - return; - } - - launchedNMTasks.addAll(schedulerState.getActiveTasksByType(NodeManagerConfiguration.NM_TASK_PREFIX)); - if (!launchedNMTasks.isEmpty()) { - LOGGER.info("{} NM(s) in active state. Not launching additional NMs", launchedNMTasks.size()); - return; - } - - for (Map.Entry<String, Integer> entry : nmInstances.entrySet()) { - LOGGER.info("Launching {} NM(s) with profile {}", entry.getValue(), entry.getKey()); - myriadOperations.flexUpCluster(profileManager.get(entry.getKey()), entry.getValue(), null); - } - } - - /** - * Create ServiceProfile for any configured service - * - * @param cfg - * @param injector - */ - private void initServiceConfigurations(MyriadConfiguration cfg, Injector injector) { - LOGGER.info("Initializing initServiceConfigurations"); - ServiceProfileManager profileManager = injector.getInstance(ServiceProfileManager.class); - TaskConstraintsManager taskConstraintsManager = injector.getInstance(TaskConstraintsManager.class); - - Map<String, ServiceConfiguration> servicesConfigs = injector.getInstance(MyriadConfiguration.class).getServiceConfigurations(); - if (servicesConfigs != null) { - for (Map.Entry<String, ServiceConfiguration> entry : servicesConfigs.entrySet()) { - final String taskPrefix = entry.getKey(); - ServiceConfiguration config = entry.getValue(); - final Double cpu = config.getCpus().or(ServiceConfiguration.DEFAULT_CPU); - final Double mem = config.getJvmMaxMemoryMB().or(ServiceConfiguration.DEFAULT_MEMORY); - - profileManager.add(new ServiceResourceProfile(taskPrefix, cpu, mem)); - taskConstraintsManager.addTaskConstraints(taskPrefix, new ServiceTaskConstraints(cfg, taskPrefix)); - } - } - } - - private void initTerminatorService(Injector injector) { - LOGGER.info("Initializing Terminator"); - terminatorService = Executors.newScheduledThreadPool(1); - final int initialDelay = 100; - final int period = 2000; - terminatorService.scheduleAtFixedRate(injector.getInstance(TaskTerminator.class), initialDelay, period, TimeUnit.MILLISECONDS); - } - - private void initRebalancerService(MyriadConfiguration cfg, Injector injector) { - if (cfg.isRebalancer()) { - LOGGER.info("Initializing Rebalancer"); - rebalancerService = Executors.newScheduledThreadPool(1); - final int initialDelay = 100; - final int period = 5000; - rebalancerService.scheduleAtFixedRate(injector.getInstance(Rebalancer.class), initialDelay, period, TimeUnit.MILLISECONDS); - } else { - LOGGER.info("Rebalancer is not turned on"); - } - } - - private void initDisruptors(Injector injector) { - LOGGER.info("Initializing Disruptors"); - DisruptorManager disruptorManager = injector.getInstance(DisruptorManager.class); - disruptorManager.init(injector); - } - - /** - * Start tasks for configured services - * - * @param injector - */ - private void startJavaBasedTaskInstance(Injector injector) { - Map<String, ServiceConfiguration> auxServicesConfigs = injector.getInstance(MyriadConfiguration.class).getServiceConfigurations(); - if (auxServicesConfigs != null) { - MyriadOperations myriadOperations = injector.getInstance(MyriadOperations.class); - for (Map.Entry<String, ServiceConfiguration> entry : auxServicesConfigs.entrySet()) { - try { - myriadOperations.flexUpAService(entry.getValue().getMaxInstances().or(1), entry.getKey()); - } catch (MyriadBadConfigurationException e) { - LOGGER.warn("Exception while trying to flexup service: {}", entry.getKey(), e); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/101bcad3/myriad-scheduler/src/main/java/com/ebay/myriad/MesosModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/MesosModule.java b/myriad-scheduler/src/main/java/com/ebay/myriad/MesosModule.java deleted file mode 100644 index d8f28c5..0000000 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/MesosModule.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package com.ebay.myriad; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.apache.mesos.MesosSchedulerDriver; -import org.apache.mesos.Protos.Credential; -import org.apache.mesos.Protos.FrameworkID; -import org.apache.mesos.Protos.FrameworkInfo; -import org.apache.mesos.Protos.FrameworkInfo.Builder; -import org.apache.mesos.SchedulerDriver; -import org.apache.mesos.state.State; -import org.apache.mesos.state.ZooKeeperState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.ebay.myriad.configuration.MyriadConfiguration; -import com.ebay.myriad.scheduler.MyriadDriver; -import com.ebay.myriad.scheduler.MyriadScheduler; -import com.ebay.myriad.state.SchedulerState; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.Singleton; -import com.google.protobuf.ByteString; - -/** - * Guice Module for Mesos objects. - */ -public class MesosModule extends AbstractModule { - private static final Logger LOGGER = LoggerFactory.getLogger(MesosModule.class); - - public MesosModule() { - } - - @Override - protected void configure() { - bind(MyriadDriver.class).in(Scopes.SINGLETON); - } - - @Provides - @Singleton - SchedulerDriver providesSchedulerDriver(MyriadScheduler scheduler, MyriadConfiguration cfg, SchedulerState schedulerState) { - - Builder frameworkInfoBuilder = FrameworkInfo.newBuilder().setUser("").setName(cfg.getFrameworkName()).setCheckpoint(cfg.isCheckpoint()).setFailoverTimeout(cfg.getFrameworkFailoverTimeout()); - - if (StringUtils.isNotEmpty(cfg.getFrameworkRole())) { - frameworkInfoBuilder.setRole(cfg.getFrameworkRole()); - } - - FrameworkID frameworkId = schedulerState.getFrameworkID(); - if (frameworkId != null) { - LOGGER.info("Attempting to re-register with frameworkId: {}", frameworkId.getValue()); - frameworkInfoBuilder.setId(frameworkId); - } - - String mesosAuthenticationPrincipal = cfg.getMesosAuthenticationPrincipal(); - String mesosAuthenticationSecretFilename = cfg.getMesosAuthenticationSecretFilename(); - if (StringUtils.isNotEmpty(mesosAuthenticationPrincipal)) { - frameworkInfoBuilder.setPrincipal(mesosAuthenticationPrincipal); - - Credential.Builder credentialBuilder = Credential.newBuilder(); - credentialBuilder.setPrincipal(mesosAuthenticationPrincipal); - if (StringUtils.isNotEmpty(mesosAuthenticationSecretFilename)) { - try { - credentialBuilder.setSecret(ByteString.readFrom(new FileInputStream(mesosAuthenticationSecretFilename))); - } catch (FileNotFoundException ex) { - LOGGER.error("Mesos authentication secret file was not found", ex); - throw new RuntimeException(ex); - } catch (IOException ex) { - LOGGER.error("Error reading Mesos authentication secret file", ex); - throw new RuntimeException(ex); - } - } - return new MesosSchedulerDriver(scheduler, frameworkInfoBuilder.build(), cfg.getMesosMaster(), credentialBuilder.build()); - } else { - return new MesosSchedulerDriver(scheduler, frameworkInfoBuilder.build(), cfg.getMesosMaster()); - } - } - - @Provides - @Singleton - State providesStateStore(MyriadConfiguration cfg) { - return new ZooKeeperState(cfg.getZkServers(), cfg.getZkTimeout(), TimeUnit.MILLISECONDS, "/myriad/" + cfg.getFrameworkName()); - } -}
