http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index 88f74ef..17f8c95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -22,7 +22,7 @@ info: Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like. - To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration. + To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration. We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management. @@ -41,177 +41,177 @@ schemes: - http - https # will be prefixed to all paths -basePath: /services/v1/ +basePath: /ws/v1/ consumes: - application/json produces: - application/json paths: - /applications: + /services: get: - summary: List of applications/services running in the cluster - description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name. + summary: List of services running in the cluster + description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. responses: 200: - description: An array of applications + description: An array of services schema: type: array items: - $ref: '#/definitions/Application' + $ref: '#/definitions/Service' default: description: Unexpected error schema: - $ref: '#/definitions/ApplicationStatus' + $ref: '#/definitions/ServiceStatus' post: - summary: Create an application/service - description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state. + summary: Create a service + description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state. parameters: - - name: Application + - name: Service in: body - description: Application request object + description: Service request object required: true schema: - $ref: '#/definitions/Application' + $ref: '#/definitions/Service' responses: 202: description: Request accepted default: description: Unexpected error schema: - $ref: '#/definitions/ApplicationStatus' + $ref: '#/definitions/ServiceStatus' - /applications/{app_name}: + /services/{service_name}: get: - summary: Get an application/service details - description: Return the details (including containers) of a running application + summary: Get service details + description: Return the details (including containers) of a running service parameters: - - name: app_name + - name: service_name in: path - description: Application name + description: Service name required: true type: string responses: 200: - description: An application object + description: a service object schema: type: object items: - $ref: '#/definitions/Application' + $ref: '#/definitions/Service' examples: - app_name: logsearch + service_name: logsearch artifact: id: logsearch:latest type: docker 404: - description: Application does not exist + description: Service does not exist default: description: Unexpected error schema: - $ref: '#/definitions/ApplicationStatus' + $ref: '#/definitions/ServiceStatus' put: - summary: Update an application/service or upgrade the binary version of the components of a running application - description: Update the runtime properties of an application. As of now, only update of lifetime and number of instances (flexing) of the components of an application is supported. The PUT operation is also used to orchestrate an upgrade of the application containers to a newer version of their artifacts. + summary: Update a service or upgrade the binary version of the components of a running service + description: Update the runtime properties of a service. As of now, only update of lifetime and number of instances (flexing) of the components of a service is supported. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts. parameters: - - name: app_name + - name: service_name in: path - description: Application name + description: Service name required: true type: string responses: 204: description: Update or upgrade was successful 404: - description: Application does not exist + description: Service does not exist default: description: Unexpected error schema: - $ref: '#/definitions/ApplicationStatus' + $ref: '#/definitions/ServiceStatus' delete: - summary: Destroy application/service - description: Destroy an application and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet. + summary: Destroy service + description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet. parameters: - - name: app_name + - name: service_name in: path - description: Application name + description: Service name required: true type: string responses: 204: description: Destroy was successful 404: - description: Application does not exist + description: Service does not exist default: description: Unexpected error schema: - $ref: '#/definitions/ApplicationStatus' + $ref: '#/definitions/ServiceStatus' definitions: - Application: - description: An Application resource has the following attributes. + Service: + description: a service resource has the following attributes. required: - name properties: name: type: string - description: A unique application name. If Registry DNS is enabled, the max length is 63 characters. + description: A unique service name. If Registry DNS is enabled, the max length is 63 characters. id: type: string - description: A unique application id. + description: A unique service id. artifact: - description: Artifact of single-component applications. + description: Artifact of single-component service. $ref: '#/definitions/Artifact' resource: - description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level. + description: Resource of single-component service or the global default for multi-component services. Mandatory if it is a single-component service and if cpus and memory are not specified at the Service level. $ref: '#/definitions/Resource' launch_command: type: string - description: The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section. + description: The custom launch command of a service component (optional). If not specified for services with docker images say, it will default to the default start command of the image. If there is a single component in this service, you can specify this without the need to have a 'components' section. launch_time: type: string format: date - description: The time when the application was created, e.g. 2016-03-16T01:01:49.000Z. + description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z. number_of_containers: type: integer format: int64 - description: Number of containers for each app-component in the application. Each app-component can further override this app-level global default. + description: Number of containers for each component in the service. Each component can further override this service-level global default. number_of_running_containers: type: integer format: int64 - description: In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests. + description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests. lifetime: type: integer format: int64 - description: Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value. + description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value. placement_policy: - description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902. + description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902. $ref: '#/definitions/PlacementPolicy' components: - description: Components of an application. + description: Components of a service. type: array items: $ref: '#/definitions/Component' configuration: - description: Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level. + description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level. $ref: '#/definitions/Configuration' containers: - description: Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application. + description: Containers of a started service. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service. type: array items: $ref: '#/definitions/Container' state: - description: State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application. - $ref: '#/definitions/ApplicationState' + description: State of the service. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started service. + $ref: '#/definitions/ServiceState' quicklinks: type: object - description: A blob of key-value pairs of quicklinks to be exported for an application. + description: A blob of key-value pairs of quicklinks to be exported for a service. additionalProperties: type: string queue: type: string - description: The YARN queue that this application should be submitted to. + description: The YARN queue that this service should be submitted to. Resource: description: - Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise. + Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise. properties: profile: type: string @@ -224,75 +224,75 @@ definitions: type: string description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB. PlacementPolicy: - description: Placement policy of an instance of an application. This feature is in the works in YARN-4902. + description: Placement policy of an instance of a service. This feature is in the works in YARN-6592. properties: label: type: string - description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc. + description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc. Artifact: - description: Artifact of an application component. If not specified, component will just run the bare launch command and no artifact will be localized. + description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized. required: - id properties: id: type: string - description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, name of application, etc. + description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc. type: type: string - description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For APPLICATION type, the application specified will be read and its components will be added into this application. The original component with artifact type APPLICATION will be removed (any properties specified in the original component will be ignored). + description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored). enum: - DOCKER - TARBALL - - APPLICATION + - SERVICE default: DOCKER uri: type: string description: Artifact location to support multiple artifact stores (optional). Component: - description: One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications. + description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services. required: - name properties: name: type: string - description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters. + description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters. dependencies: type: array items: type: string - description: An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG. + description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG. readiness_check: - description: Readiness check for this app-component. + description: Readiness check for this component. $ref: '#/definitions/ReadinessCheck' artifact: - description: Artifact of the component (optional). If not specified, the application level global artifact takes effect. + description: Artifact of the component (optional). If not specified, the service level global artifact takes effect. $ref: '#/definitions/Artifact' launch_command: type: string description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any). resource: - description: Resource of this component (optional). If not specified, the application level global resource takes effect. + description: Resource of this component (optional). If not specified, the service level global resource takes effect. $ref: '#/definitions/Resource' number_of_containers: type: integer format: int64 - description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect. + description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect. run_privileged_container: type: boolean description: Run all containers of this component in privileged mode (YARN-4262). placement_policy: - description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details. + description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details. $ref: '#/definitions/PlacementPolicy' configuration: - description: Config properties for this app-component. + description: Config properties for this component. $ref: '#/definitions/Configuration' quicklinks: type: array items: type: string - description: A list of quicklink keys defined at the application level, and to be resolved by this component. + description: A list of quicklink keys defined at the service level, and to be resolved by this component. ReadinessCheck: - description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases. + description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases. required: - type properties: @@ -308,28 +308,28 @@ definitions: additionalProperties: type: string artifact: - description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET + description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET $ref: '#/definitions/Artifact' Configuration: - description: Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported. + description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported. properties: properties: type: object - description: A blob of key-value pairs of common application properties. + description: A blob of key-value pairs of common service properties. additionalProperties: type: string env: type: object - description: A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection. + description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection. additionalProperties: type: string files: - description: Array of list of files that needs to be created and made available as volumes in the application component containers. + description: Array of list of files that needs to be created and made available as volumes in the service component containers. type: array items: $ref: '#/definitions/ConfigFile' ConfigFile: - description: A config file that needs to be created and made available as a volume in an application component container. + description: A config file that needs to be created and made available as a volume in a service component container. properties: type: type: string @@ -352,11 +352,11 @@ definitions: type: object description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file. Container: - description: An instance of a running application container. + description: An instance of a running service container. properties: id: type: string - description: Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002. + description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002. launch_time: type: string format: date @@ -371,7 +371,7 @@ definitions: type: string description: The bare node or host in which the container is running, e.g. cn008.example.com. state: - description: State of the container of an application. + description: State of the container of a service. $ref: '#/definitions/ContainerState' component_name: type: string @@ -385,12 +385,12 @@ definitions: privileged_container: type: boolean description: Container running in privileged mode or not. - ApplicationState: - description: The current state of an application. + ServiceState: + description: The current state of a service. properties: state: type: string - description: enum of the state of the application + description: enum of the state of the service enum: - ACCEPTED - STARTED @@ -398,25 +398,26 @@ definitions: - STOPPED - FAILED ContainerState: - description: The current state of the container of an application. + description: The current state of the container of a service. properties: state: type: string description: enum of the state of the container enum: - INIT + - STARTED - READY - ApplicationStatus: - description: The current status of a submitted application, returned as a response to the GET API. + ServiceStatus: + description: The current status of a submitted service, returned as a response to the GET API. properties: diagnostics: type: string - description: Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state. + description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state. state: - description: Application state. - $ref: '#/definitions/ApplicationState' + description: Service state. + $ref: '#/definitions/ServiceState' code: type: integer format: int32 - description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information. + description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh deleted file mode 100644 index 9f15b7e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export SLIDER_VERSION=${project.version} -export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION} -export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" -export LIB_PARENT_DIR=`dirname $SCRIPT_DIR` -export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40} -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf} -export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191} -export APP_RUNAS_USER=${APP_RUNAS_USER:-root} -export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/} -export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log" -$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000..2814cca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/dev-support/findbugs-exclude.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<FindBugsFilter> + <Match> + <Package name="org.apache.hadoop.yarn.proto" /> + </Match> + <Match> + <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" /> + <Bug pattern="MS_CANNOT_BE_FINAL" /> + </Match> + <Match> + <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" /> + <Bug pattern="OBL_UNSATISFIED_OBLIGATION" /> + </Match> + <Match> + <Class name="org.apache.hadoop.yarn.service.utils.JsonSerDeser" /> + <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" /> + </Match> + <Match> + <Package name="org.apache.hadoop.yarn.service.client.params"/> + <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/> + </Match> + <Match> + <Package name="org.apache.hadoop.yarn.service.client.params"/> + <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/> + </Match> + <Match> + <Class name="org.apache.hadoop.yarn.service.client.ServiceClient"/> + <Field name="registryClient" /> + <Bug pattern="IS2_INCONSISTENT_SYNC"/> + </Match> + +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties new file mode 100644 index 0000000..58c8e27 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf/yarnservice-log4j.properties @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for Slider Application Master + +# Log rotation based on size (256MB) with a max of 20 backup files +log4j.rootLogger=INFO, amlog +log4j.threshhold=ALL +log4j.appender.amlog=org.apache.log4j.RollingFileAppender +log4j.appender.amlog.layout=org.apache.log4j.PatternLayout +log4j.appender.amlog.File=${LOG_DIR}/serviceam.log +log4j.appender.amlog.MaxFileSize=256MB +log4j.appender.amlog.MaxBackupIndex=20 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n + +# for debugging yarn-service framework +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml new file mode 100644 index 0000000..1f8a408 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -0,0 +1,408 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-services</artifactId> + <version>3.0.0-beta1-SNAPSHOT</version> + </parent> + <artifactId>hadoop-yarn-services-core</artifactId> + <packaging>jar</packaging> + <name>Apache Hadoop YARN Services Core</name> + + <properties> + <!-- Needed for generating FindBugs warnings using parent pom --> + <yarn.basedir>${project.parent.basedir}</yarn.basedir> + </properties> + + <build> + <!-- resources are filtered for dynamic updates. This gets build info in--> + <resources> + <resource> + <directory>src/main/resources</directory> + <filtering>true</filtering> + </resource> + </resources> + + <plugins> + <plugin> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> + <executions> + <execution> + <id>compile-protoc</id> + <goals> + <goal>protoc</goal> + </goals> + <configuration> + <protocVersion>${protobuf.version}</protocVersion> + <protocCommand>${protoc.path}</protocCommand> + <imports> + <param>${basedir}/src/main/proto</param> + </imports> + <source> + <directory>${basedir}/src/main/proto</directory> + <includes> + <include>ClientAMProtocol.proto</include> + </includes> + </source> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <environmentVariables> + <JAVA_HOME>${java.home}</JAVA_HOME> + </environmentVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/*.json</exclude> + </excludes> + </configuration> + </plugin> + + </plugins> + </build> + <dependencies> + <dependency> + <groupId>com.beust</groupId> + <artifactId>jcommander</artifactId> + <version>1.30</version> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs-client</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-registry</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + </dependency> + + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + <version>1.8</version> + </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + + <dependency> + <groupId>com.codahale.metrics</groupId> + <artifactId>metrics-core</artifactId> + </dependency> + + <dependency> + <groupId>com.codahale.metrics</groupId> + <artifactId>metrics-servlets</artifactId> + <version>3.0.1</version> + </dependency> + + <!-- ======================================================== --> + <!-- service registry --> + <!-- ======================================================== --> + + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </dependency> + + <!-- ======================================================== --> + <!-- Jersey and webapp support --> + <!-- ======================================================== --> + + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </dependency> + + <dependency> + <groupId>javax.xml.bind</groupId> + <artifactId>jaxb-api</artifactId> + </dependency> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </dependency> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </dependency> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </dependency> + + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + </dependency> + + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + + <dependency> + <groupId>com.google.inject.extensions</groupId> + <artifactId>guice-servlet</artifactId> + </dependency> + + <dependency> + <groupId>com.sun.jersey.contribs</groupId> + <artifactId>jersey-guice</artifactId> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <version>3.1</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.objenesis</groupId> + <artifactId>objenesis</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-easymock</artifactId> + <version>1.6.5</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>1.6.5</version> + <exclusions> + <exclusion> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + </exclusion> + <exclusion> + <groupId>org.objenesis</groupId> + <artifactId>objenesis</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>javax.servlet.jsp</groupId> + <artifactId>jsp-api</artifactId> + <scope>runtime</scope> + </dependency> + + <dependency> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + </dependency> + + <dependency> + <groupId>org.yaml</groupId> + <artifactId>snakeyaml</artifactId> + <version>1.16</version> + <scope>compile</scope> + </dependency> + + <dependency> + <groupId>io.swagger</groupId> + <artifactId>swagger-annotations</artifactId> + <version>1.5.4</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <scope>test</scope> + </dependency> + + </dependencies> + + + <profiles> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <finalName>${project.artifactId}-${project.version}</finalName> + <descriptorRefs> + <descriptorRef>hadoop-yarn-services-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + </profiles> + +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java new file mode 100644 index 0000000..516d23d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; + +import java.io.IOException; + +public interface ClientAMProtocol { + FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request) + throws IOException, YarnException; + + GetStatusResponseProto getStatus(GetStatusRequestProto requestProto) + throws IOException, YarnException; + + StopResponseProto stop(StopRequestProto requestProto) + throws IOException, YarnException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java new file mode 100644 index 0000000..8e4c34d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto; +import org.apache.hadoop.yarn.service.component.ComponentEvent; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX; + +public class ClientAMService extends AbstractService + implements ClientAMProtocol { + + private static final Logger LOG = + LoggerFactory.getLogger(ClientAMService.class); + + private ServiceContext context; + private Server server; + + private InetSocketAddress bindAddress; + + public ClientAMService(ServiceContext context) { + super("Client AM Service"); + this.context = context; + } + + @Override protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress address = new InetSocketAddress(0); + server = rpc.getServer(ClientAMProtocol.class, this, address, conf, + context.secretManager, 1); + server.start(); + + String nodeHostString = + System.getenv(ApplicationConstants.Environment.NM_HOST.name()); + + bindAddress = NetUtils.createSocketAddrForHost(nodeHostString, + server.getListenerAddress().getPort()); + + LOG.info("Instantiated ClientAMService at " + bindAddress); + super.serviceStart(); + } + + @Override protected void serviceStop() throws Exception { + if (server != null) { + server.stop(); + } + super.serviceStop(); + } + + @Override public FlexComponentsResponseProto flexComponents( + FlexComponentsRequestProto request) throws IOException { + if (!request.getComponentsList().isEmpty()) { + for (ComponentCountProto component : request.getComponentsList()) { + ComponentEvent event = new ComponentEvent(component.getName(), FLEX) + .setDesired(component.getNumberOfContainers()); + context.scheduler.getDispatcher().getEventHandler().handle(event); + LOG.info("Flexing component {} to {}", component.getName(), + component.getNumberOfContainers()); + } + } + return FlexComponentsResponseProto.newBuilder().build(); + } + + @Override + public GetStatusResponseProto getStatus(GetStatusRequestProto request) + throws IOException, YarnException { + String stat = ServiceApiUtil.jsonSerDeser.toJson(context.service); + return GetStatusResponseProto.newBuilder().setStatus(stat).build(); + } + + @Override + public StopResponseProto stop(StopRequestProto requestProto) + throws IOException, YarnException { + LOG.info("Stop the service."); + // Stop the service in 2 seconds delay to make sure this rpc call is completed. + // shutdown hook will be executed which will stop AM gracefully. + Thread thread = new Thread() { + @Override + public void run() { + try { + Thread.sleep(2000); + ExitUtil.terminate(0); + } catch (InterruptedException e) { + LOG.error("Interrupted while stopping", e); + } + } + }; + thread.start(); + return StopResponseProto.newBuilder().build(); + } + + public InetSocketAddress getBindAddress() { + return bindAddress; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java new file mode 100644 index 0000000..4743f28 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.yarn.service.component.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD; + +/** + * This tracks the container failures per node. If the failure counter exceeds + * the maxFailurePerNode limit, it'll blacklist that node. + * + */ +public class ContainerFailureTracker { + + private static final Logger LOG = + LoggerFactory.getLogger(ContainerFailureTracker.class); + + // Host -> num container failures + private Map<String, Integer> failureCountPerNode = new HashMap<>(); + private Set<String> blackListedNodes = new HashSet<>(); + private ServiceContext context; + private int maxFailurePerNode; + private Component component; + + public ContainerFailureTracker(ServiceContext context, Component component) { + this.context = context; + this.component = component; + maxFailurePerNode = component.getComponentSpec().getConfiguration() + .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3); + } + + + public synchronized void incNodeFailure(String host) { + int num = 0; + if (failureCountPerNode.containsKey(host)) { + num = failureCountPerNode.get(host); + } + num++; + failureCountPerNode.put(host, num); + + // black list the node if exceed max failure + if (num > maxFailurePerNode && !blackListedNodes.contains(host)) { + List<String> blacklists = new ArrayList<>(); + blacklists.add(host); + blackListedNodes.add(host); + context.scheduler.getAmRMClient().updateBlacklist(blacklists, null); + LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}." + + " Current list of blacklisted nodes: {}", + component.getName(), num, host, blackListedNodes); + } + } + + public synchronized void resetContainerFailures() { + // reset container failure counter per node + failureCountPerNode.clear(); + context.scheduler.getAmRMClient() + .updateBlacklist(null, new ArrayList<>(blackListedNodes)); + LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ", + component.getName(), blackListedNodes); + blackListedNodes.clear(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java new file mode 100644 index 0000000..94dbc6e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import com.google.common.cache.LoadingCache; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; + +public class ServiceContext { + public Service service = null; + public SliderFileSystem fs; + public String serviceHdfsDir = ""; + public ApplicationAttemptId attemptId; + public LoadingCache<ConfigFile, Object> configCache; + public ServiceScheduler scheduler; + public ClientToAMTokenSecretManager secretManager; + public ClientAMService clientAMService; + + public ServiceContext() { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java new file mode 100644 index 0000000..e700961 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +public class ServiceMaster extends CompositeService { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceMaster.class); + + private static SliderAMArgs amArgs; + protected ServiceContext context; + + public ServiceMaster(String name) { + super(name); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + //TODO Deprecate slider conf, make sure works with yarn conf + printSystemEnv(); + if (UserGroupInformation.isSecurityEnabled()) { + UserGroupInformation.setConfiguration(conf); + } + LOG.info("Login user is {}", UserGroupInformation.getLoginUser()); + + context = new ServiceContext(); + Path appDir = getAppDir(); + context.serviceHdfsDir = appDir.toString(); + SliderFileSystem fs = new SliderFileSystem(conf); + context.fs = fs; + fs.setAppDir(appDir); + loadApplicationJson(context, fs); + + ContainerId amContainerId = getAMContainerId(); + + ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId(); + LOG.info("Service AppAttemptId: " + attemptId); + context.attemptId = attemptId; + + // configure AM to wait forever for RM + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1); + conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS); + + DefaultMetricsSystem.initialize("ServiceAppMaster"); + + context.secretManager = new ClientToAMTokenSecretManager(attemptId, null); + ClientAMService clientAMService = new ClientAMService(context); + context.clientAMService = clientAMService; + addService(clientAMService); + + ServiceScheduler scheduler = createServiceScheduler(context); + addService(scheduler); + context.scheduler = scheduler; + + ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context); + addService(monitor); + + super.serviceInit(conf); + } + + protected ContainerId getAMContainerId() throws BadClusterStateException { + return ContainerId.fromString(SliderUtils.mandatoryEnvVariable( + ApplicationConstants.Environment.CONTAINER_ID.name())); + } + + protected Path getAppDir() { + return new Path(amArgs.getAppDefPath()).getParent(); + } + + protected ServiceScheduler createServiceScheduler(ServiceContext context) + throws IOException, YarnException { + return new ServiceScheduler(context); + } + + protected void loadApplicationJson(ServiceContext context, + SliderFileSystem fs) throws IOException { + context.service = ServiceApiUtil + .loadServiceFrom(fs, new Path(amArgs.getAppDefPath())); + LOG.info(context.service.toString()); + } + + @Override + protected void serviceStop() throws Exception { + LOG.info("Stopping app master"); + super.serviceStop(); + } + + private void printSystemEnv() { + for (Map.Entry<String, String> envs : System.getenv().entrySet()) { + LOG.info("{} = {}", envs.getKey(), envs.getValue()); + } + } + + public static void main(String[] args) throws Exception { + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG); + amArgs = new SliderAMArgs(args); + amArgs.parse(); + try { + ServiceMaster serviceMaster = new ServiceMaster("Service Master"); + ShutdownHookManager.get() + .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30); + YarnConfiguration conf = new YarnConfiguration(); + new GenericOptionsParser(conf, args); + serviceMaster.init(conf); + serviceMaster.start(); + } catch (Throwable t) { + LOG.error("Error starting service master", t); + ExitUtil.terminate(1, "Error starting service master"); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java new file mode 100644 index 0000000..9fc886e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; + +import static org.apache.hadoop.metrics2.lib.Interns.info; + +@Metrics(context = "yarn-native-service") +public class ServiceMetrics implements MetricsSource { + + @Metric("containers requested") + public MutableGaugeInt containersRequested; + + @Metric("anti-affinity containers pending") + public MutableGaugeInt pendingAAContainers; + + @Metric("containers running") + public MutableGaugeInt containersRunning; + + @Metric("containers ready") + public MutableGaugeInt containersReady; + + @Metric("containers desired") + public MutableGaugeInt containersDesired; + + @Metric("containers succeeded") + public MutableGaugeInt containersSucceeded; + + @Metric("containers failed") + public MutableGaugeInt containersFailed; + + @Metric("containers preempted") + public MutableGaugeInt containersPreempted; + + @Metric("containers surplus") + public MutableGaugeInt surplusContainers; + + @Metric("containers failed due to disk failure") + public MutableGaugeInt containersDiskFailure; + + protected final MetricsRegistry registry; + + public ServiceMetrics(MetricsInfo metricsInfo) { + registry = new MetricsRegistry(metricsInfo); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + registry.snapshot(collector.addRecord(registry.info()), all); + } + + public static ServiceMetrics register(String name, String description) { + ServiceMetrics metrics = new ServiceMetrics(info(name, description)); + DefaultMetricsSystem.instance().register(name, description, metrics); + return metrics; + } + + public void tag(String name, String description, String value) { + registry.tag(name, description, value); + } + + @Override public String toString() { + return "ServiceMetrics{" + + "containersRequested=" + containersRequested.value() + + ", pendingAAContainers=" + pendingAAContainers.value() + + ", containersRunning=" + containersRunning.value() + + ", containersDesired=" + containersDesired.value() + + ", containersSucceeded=" + containersSucceeded.value() + + ", containersFailed=" + containersFailed.value() + + ", containersPreempted=" + containersPreempted.value() + + ", surplusContainers=" + surplusContainers.value() + '}'; + } +} + --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
