YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3928a646 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3928a646 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3928a646 Branch: refs/heads/yarn-native-services Commit: 3928a646a9bc3be6dcdb117e2bdcb249bc3695ea Parents: 9c4d98b Author: Billie Rinaldi <bil...@apache.org> Authored: Mon Aug 28 09:59:55 2017 -0700 Committer: Jian He <jia...@apache.org> Committed: Fri Oct 13 11:01:52 2017 -0700 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 8 +- .../assemblies/hadoop-yarn-services-dist.xml | 2 +- hadoop-project/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 14 +- .../hadoop-yarn/conf/yarn-env.sh | 2 +- .../hadoop-yarn-services-api/pom.xml | 2 +- .../hadoop/yarn/service/webapp/ApiServer.java | 274 +++ .../yarn/service/webapp/ApiServerWebApp.java | 159 ++ .../service/webapp/ApplicationApiService.java | 275 --- .../service/webapp/ApplicationApiWebApp.java | 123 -- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 185 +- .../src/main/scripts/run_rest_service.sh | 28 - .../dev-support/findbugs-exclude.xml | 48 + .../conf/yarnservice-log4j.properties | 62 + .../hadoop-yarn-services-core/pom.xml | 408 +++++ .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java | 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 156 ++ .../hadoop/yarn/service/ServiceMetrics.java | 98 + .../hadoop/yarn/service/ServiceScheduler.java | 654 +++++++ .../yarn/service/api/ServiceApiConstants.java | 69 + .../yarn/service/api/records/Artifact.java | 160 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 412 +++++ .../yarn/service/api/records/ConfigFile.java | 225 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 297 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java | 102 ++ .../service/api/records/ReadinessCheck.java | 175 ++ .../yarn/service/api/records/Resource.java | 159 ++ .../yarn/service/api/records/Service.java | 466 +++++ .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../hadoop/yarn/service/client/ServiceCLI.java | 112 ++ .../yarn/service/client/ServiceClient.java | 892 +++++++++ .../client/params/AbstractActionArgs.java | 158 ++ .../client/params/AbstractArgsDelegate.java | 28 + .../AbstractClusterBuildingActionArgs.java | 58 + .../service/client/params/ActionBuildArgs.java | 31 + .../service/client/params/ActionClientArgs.java | 71 + .../service/client/params/ActionCreateArgs.java | 33 + .../client/params/ActionDependencyArgs.java | 65 + .../client/params/ActionDestroyArgs.java | 37 + .../service/client/params/ActionExistsArgs.java | 49 + .../service/client/params/ActionFlexArgs.java | 50 + .../service/client/params/ActionFreezeArgs.java | 56 + .../service/client/params/ActionHelpArgs.java | 44 + .../service/client/params/ActionKDiagArgs.java | 76 + .../service/client/params/ActionKeytabArgs.java | 76 + .../service/client/params/ActionListArgs.java | 76 + .../client/params/ActionRegistryArgs.java | 218 +++ .../client/params/ActionResolveArgs.java | 153 ++ .../client/params/ActionResourceArgs.java | 70 + .../service/client/params/ActionStatusArgs.java | 51 + .../service/client/params/ActionThawArgs.java | 67 + .../service/client/params/ActionTokensArgs.java | 78 + .../service/client/params/ActionUpdateArgs.java | 32 + .../yarn/service/client/params/ArgOps.java | 156 ++ .../yarn/service/client/params/Arguments.java | 103 ++ .../yarn/service/client/params/ClientArgs.java | 252 +++ .../yarn/service/client/params/CommonArgs.java | 282 +++ .../client/params/ComponentArgsDelegate.java | 52 + .../client/params/DontSplitArguments.java | 34 + .../client/params/LaunchArgsAccessor.java | 30 + .../client/params/LaunchArgsDelegate.java | 51 + .../client/params/OptionArgsDelegate.java | 66 + .../client/params/PathArgumentConverter.java | 34 + .../service/client/params/SliderAMArgs.java | 57 + .../client/params/SliderAMCreateAction.java | 73 + .../service/client/params/SliderActions.java | 82 + .../service/client/params/WaitArgsDelegate.java | 42 + .../service/client/params/WaitTimeAccessor.java | 24 + .../yarn/service/component/Component.java | 494 +++++ .../yarn/service/component/ComponentEvent.java | 83 + .../service/component/ComponentEventType.java | 26 + .../yarn/service/component/ComponentState.java | 25 + .../component/instance/ComponentInstance.java | 493 +++++ .../instance/ComponentInstanceEvent.java | 58 + .../instance/ComponentInstanceEventType.java | 26 + .../component/instance/ComponentInstanceId.java | 91 + .../instance/ComponentInstanceState.java | 26 + .../yarn/service/conf/RestApiConstants.java | 43 + .../yarn/service/conf/SliderExitCodes.java | 88 + .../yarn/service/conf/YarnServiceConf.java | 101 ++ .../yarn/service/conf/YarnServiceConstants.java | 90 + .../containerlaunch/AbstractLauncher.java | 271 +++ .../containerlaunch/ClasspathConstructor.java | 172 ++ .../containerlaunch/CommandLineBuilder.java | 86 + .../containerlaunch/ContainerLaunchService.java | 101 ++ .../containerlaunch/CredentialUtils.java | 319 ++++ .../containerlaunch/JavaCommandLineBuilder.java | 181 ++ .../exceptions/BadClusterStateException.java | 36 + .../BadCommandArgumentsException.java | 30 + .../service/exceptions/BadConfigException.java | 39 + .../yarn/service/exceptions/ErrorStrings.java | 42 + .../service/exceptions/ExitCodeProvider.java | 32 + .../service/exceptions/LauncherExitCodes.java | 196 ++ .../exceptions/RestApiErrorMessages.java | 92 + .../exceptions/ServiceLaunchException.java | 73 + .../service/exceptions/SliderException.java | 66 + .../yarn/service/exceptions/UsageException.java | 34 + .../pb/client/ClientAMProtocolPBClientImpl.java | 91 + .../impl/pb/service/ClientAMProtocolPB.java | 29 + .../service/ClientAMProtocolPBServiceImpl.java | 70 + .../yarn/service/monitor/ServiceMonitor.java | 147 ++ .../yarn/service/monitor/probe/HttpProbe.java | 110 ++ .../service/monitor/probe/LogEntryBuilder.java | 76 + .../yarn/service/monitor/probe/MonitorKeys.java | 66 + .../service/monitor/probe/MonitorUtils.java | 84 + .../yarn/service/monitor/probe/PortProbe.java | 98 + .../yarn/service/monitor/probe/Probe.java | 100 ++ .../yarn/service/monitor/probe/ProbeStatus.java | 160 ++ .../provider/AbstractClientProvider.java | 122 ++ .../provider/AbstractProviderService.java | 109 ++ .../yarn/service/provider/ProviderFactory.java | 76 + .../yarn/service/provider/ProviderService.java | 39 + .../yarn/service/provider/ProviderUtils.java | 402 +++++ .../defaultImpl/DefaultClientProvider.java | 46 + .../defaultImpl/DefaultProviderFactory.java | 51 + .../defaultImpl/DefaultProviderService.java | 36 + .../provider/docker/DockerClientProvider.java | 53 + .../service/provider/docker/DockerKeys.java | 30 + .../provider/docker/DockerProviderFactory.java | 52 + .../provider/docker/DockerProviderService.java | 57 + .../provider/tarball/TarballClientProvider.java | 65 + .../tarball/TarballProviderFactory.java | 52 + .../tarball/TarballProviderService.java | 48 + .../registry/CustomRegistryConstants.java | 57 + .../registry/YarnRegistryViewForProviders.java | 225 +++ .../timelineservice/ServiceMetricsSink.java | 102 ++ .../ServiceTimelineEntityType.java | 39 + .../timelineservice/ServiceTimelineEvent.java | 34 + .../ServiceTimelineMetricsConstants.java | 92 + .../ServiceTimelinePublisher.java | 368 ++++ .../service/timelineservice/package-info.java | 27 + .../utils/ApplicationReportSerDeser.java | 56 + .../service/utils/ClientRegistryBinder.java | 201 +++ .../hadoop/yarn/service/utils/Comparators.java | 62 + .../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++ .../hadoop/yarn/service/utils/ConfigUtils.java | 97 + .../yarn/service/utils/CoreFileSystem.java | 521 ++++++ .../hadoop/yarn/service/utils/Duration.java | 109 ++ .../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++ .../yarn/service/utils/KerberosDiags.java | 680 +++++++ .../yarn/service/utils/PatternValidator.java | 58 + .../hadoop/yarn/service/utils/PortScanner.java | 113 ++ .../service/utils/PublishedConfiguration.java | 196 ++ .../utils/PublishedConfigurationOutputter.java | 212 +++ .../utils/SerializedApplicationReport.java | 98 + .../yarn/service/utils/ServiceApiUtil.java | 446 +++++ .../service/utils/ServiceRegistryUtils.java | 71 + .../yarn/service/utils/SliderFileSystem.java | 51 + .../hadoop/yarn/service/utils/SliderUtils.java | 1654 +++++++++++++++++ .../yarn/service/utils/ZookeeperUtils.java | 146 ++ .../src/main/proto/ClientAMProtocol.proto | 56 + .../hadoop/yarn/service/MockServiceAM.java | 221 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 59 + .../hadoop/yarn/service/TestServiceApiUtil.java | 530 ++++++ .../yarn/service/TestYarnNativeServices.java | 472 +++++ .../client/TestBuildExternalComponents.java | 128 ++ .../yarn/service/client/TestServiceCLI.java | 155 ++ .../yarn/service/conf/ExampleAppJson.java | 65 + .../yarn/service/conf/TestAppJsonResolve.java | 222 +++ .../service/conf/TestLoadExampleAppJson.java | 78 + .../service/conf/TestValidateServiceNames.java | 126 ++ .../service/monitor/TestServiceMonitor.java | 104 ++ .../providers/TestAbstractClientProvider.java | 118 ++ .../service/providers/TestProviderFactory.java | 76 + .../TestServiceTimelinePublisher.java | 293 +++ .../src/test/resources/example-app.json | 15 + .../service/conf/examples/app-override.json | 72 + .../hadoop/yarn/service/conf/examples/app.json | 48 + .../yarn/service/conf/examples/default.json | 16 + .../yarn/service/conf/examples/external0.json | 8 + .../yarn/service/conf/examples/external1.json | 30 + .../yarn/service/conf/examples/external2.json | 22 + .../src/test/resources/yarn-site.xml | 19 + .../hadoop-yarn-services/pom.xml | 38 + .../dev-support/findbugs-exclude.xml | 48 - .../conf/yarnservice-log4j.properties | 62 - .../hadoop-yarn-slider-core/pom.xml | 409 ----- .../src/assembly/executable-jar.xml | 47 - .../hadoop/yarn/service/ClientAMProtocol.java | 40 - .../hadoop/yarn/service/ClientAMService.java | 132 -- .../yarn/service/ContainerFailureTracker.java | 89 - .../hadoop/yarn/service/ServiceContext.java | 41 - .../hadoop/yarn/service/ServiceMaster.java | 157 -- .../hadoop/yarn/service/ServiceScheduler.java | 655 ------- .../api/constants/ServiceApiConstants.java | 69 - .../yarn/service/api/records/Application.java | 466 ----- .../service/api/records/ApplicationState.java | 33 - .../service/api/records/ApplicationStatus.java | 148 -- .../yarn/service/api/records/Artifact.java | 160 -- .../yarn/service/api/records/BaseResource.java | 52 - .../yarn/service/api/records/Component.java | 412 ----- .../yarn/service/api/records/ConfigFile.java | 225 --- .../yarn/service/api/records/ConfigFormat.java | 67 - .../yarn/service/api/records/Configuration.java | 225 --- .../yarn/service/api/records/Container.java | 297 --- .../service/api/records/ContainerState.java | 30 - .../hadoop/yarn/service/api/records/Error.java | 129 -- .../service/api/records/PlacementPolicy.java | 102 -- .../service/api/records/ReadinessCheck.java | 175 -- .../yarn/service/api/records/Resource.java | 159 -- .../yarn/service/client/ClientAMProxy.java | 49 - .../hadoop/yarn/service/client/ServiceCLI.java | 104 -- .../yarn/service/client/ServiceClient.java | 872 --------- .../client/params/AbstractActionArgs.java | 158 -- .../client/params/AbstractArgsDelegate.java | 28 - .../AbstractClusterBuildingActionArgs.java | 58 - .../service/client/params/ActionBuildArgs.java | 31 - .../service/client/params/ActionClientArgs.java | 71 - .../service/client/params/ActionCreateArgs.java | 33 - .../client/params/ActionDependencyArgs.java | 65 - .../client/params/ActionDestroyArgs.java | 37 - .../service/client/params/ActionExistsArgs.java | 49 - .../service/client/params/ActionFlexArgs.java | 50 - .../service/client/params/ActionFreezeArgs.java | 56 - .../service/client/params/ActionHelpArgs.java | 44 - .../service/client/params/ActionKDiagArgs.java | 76 - .../service/client/params/ActionKeytabArgs.java | 76 - .../service/client/params/ActionListArgs.java | 76 - .../client/params/ActionRegistryArgs.java | 218 --- .../client/params/ActionResolveArgs.java | 153 -- .../client/params/ActionResourceArgs.java | 70 - .../service/client/params/ActionStatusArgs.java | 51 - .../service/client/params/ActionThawArgs.java | 67 - .../service/client/params/ActionTokensArgs.java | 78 - .../service/client/params/ActionUpdateArgs.java | 32 - .../yarn/service/client/params/ArgOps.java | 156 -- .../yarn/service/client/params/Arguments.java | 103 -- .../yarn/service/client/params/ClientArgs.java | 252 --- .../yarn/service/client/params/CommonArgs.java | 282 --- .../client/params/ComponentArgsDelegate.java | 52 - .../client/params/DontSplitArguments.java | 34 - .../client/params/LaunchArgsAccessor.java | 30 - .../client/params/LaunchArgsDelegate.java | 51 - .../client/params/OptionArgsDelegate.java | 66 - .../client/params/PathArgumentConverter.java | 34 - .../service/client/params/SliderAMArgs.java | 57 - .../client/params/SliderAMCreateAction.java | 73 - .../service/client/params/SliderActions.java | 82 - .../service/client/params/WaitArgsDelegate.java | 42 - .../service/client/params/WaitTimeAccessor.java | 24 - .../service/compinstance/ComponentInstance.java | 493 ----- .../compinstance/ComponentInstanceEvent.java | 58 - .../ComponentInstanceEventType.java | 27 - .../compinstance/ComponentInstanceId.java | 91 - .../compinstance/ComponentInstanceState.java | 26 - .../yarn/service/component/Component.java | 493 ----- .../yarn/service/component/ComponentEvent.java | 83 - .../service/component/ComponentEventType.java | 26 - .../yarn/service/component/ComponentState.java | 25 - .../yarn/service/conf/RestApiConstants.java | 43 - .../yarn/service/conf/SliderExitCodes.java | 88 - .../yarn/service/conf/YarnServiceConf.java | 97 - .../yarn/service/conf/YarnServiceConstants.java | 90 - .../containerlaunch/AbstractLauncher.java | 271 --- .../containerlaunch/ClasspathConstructor.java | 172 -- .../containerlaunch/CommandLineBuilder.java | 86 - .../containerlaunch/ContainerLaunchService.java | 101 -- .../containerlaunch/CredentialUtils.java | 319 ---- .../containerlaunch/JavaCommandLineBuilder.java | 181 -- .../exceptions/BadClusterStateException.java | 36 - .../BadCommandArgumentsException.java | 30 - .../service/exceptions/BadConfigException.java | 39 - .../yarn/service/exceptions/ErrorStrings.java | 57 - .../service/exceptions/ExitCodeProvider.java | 32 - .../service/exceptions/LauncherExitCodes.java | 196 -- .../exceptions/RestApiErrorMessages.java | 92 - .../exceptions/ServiceLaunchException.java | 73 - .../service/exceptions/SliderException.java | 66 - .../yarn/service/exceptions/UsageException.java | 34 - .../pb/client/ClientAMProtocolPBClientImpl.java | 91 - .../impl/pb/service/ClientAMProtocolPB.java | 29 - .../service/ClientAMProtocolPBServiceImpl.java | 70 - .../yarn/service/metrics/ServiceMetrics.java | 101 -- .../provider/AbstractClientProvider.java | 122 -- .../provider/AbstractProviderService.java | 109 -- .../yarn/service/provider/ProviderFactory.java | 76 - .../yarn/service/provider/ProviderService.java | 39 - .../yarn/service/provider/ProviderUtils.java | 402 ----- .../defaultImpl/DefaultClientProvider.java | 46 - .../defaultImpl/DefaultProviderFactory.java | 51 - .../defaultImpl/DefaultProviderService.java | 36 - .../provider/docker/DockerClientProvider.java | 53 - .../service/provider/docker/DockerKeys.java | 30 - .../provider/docker/DockerProviderFactory.java | 52 - .../provider/docker/DockerProviderService.java | 57 - .../provider/tarball/TarballClientProvider.java | 65 - .../tarball/TarballProviderFactory.java | 52 - .../tarball/TarballProviderService.java | 48 - .../registry/CustomRegistryConstants.java | 57 - .../registry/YarnRegistryViewForProviders.java | 225 --- .../yarn/service/rest/BaseRestClient.java | 149 -- .../yarn/service/rest/ExceptionConverter.java | 128 -- .../hadoop/yarn/service/rest/HttpVerb.java | 57 - .../rest/SliderURLConnectionFactory.java | 176 -- .../yarn/service/rest/UgiJerseyBinding.java | 153 -- .../service/rest/UrlConnectionOperations.java | 83 - .../service/servicemonitor/ServiceMonitor.java | 148 -- .../service/servicemonitor/probe/HttpProbe.java | 110 -- .../servicemonitor/probe/LogEntryBuilder.java | 76 - .../servicemonitor/probe/MonitorKeys.java | 66 - .../servicemonitor/probe/MonitorUtils.java | 84 - .../service/servicemonitor/probe/PortProbe.java | 98 - .../service/servicemonitor/probe/Probe.java | 100 -- .../servicemonitor/probe/ProbeStatus.java | 160 -- .../timelineservice/ServiceMetricsSink.java | 102 -- .../ServiceTimelineEntityType.java | 39 - .../timelineservice/ServiceTimelineEvent.java | 34 - .../ServiceTimelineMetricsConstants.java | 92 - .../ServiceTimelinePublisher.java | 368 ---- .../service/timelineservice/package-info.java | 27 - .../utils/ApplicationReportSerDeser.java | 56 - .../service/utils/ClientRegistryBinder.java | 201 --- .../hadoop/yarn/service/utils/Comparators.java | 62 - .../hadoop/yarn/service/utils/ConfigHelper.java | 157 -- .../hadoop/yarn/service/utils/ConfigUtils.java | 97 - .../yarn/service/utils/CoreFileSystem.java | 521 ------ .../hadoop/yarn/service/utils/Duration.java | 109 -- .../hadoop/yarn/service/utils/JsonSerDeser.java | 249 --- .../yarn/service/utils/KerberosDiags.java | 680 ------- .../yarn/service/utils/PatternValidator.java | 58 - .../hadoop/yarn/service/utils/PortScanner.java | 113 -- .../service/utils/PublishedConfiguration.java | 196 -- .../utils/PublishedConfigurationOutputter.java | 212 --- .../utils/SerializedApplicationReport.java | 98 - .../yarn/service/utils/ServiceApiUtil.java | 443 ----- .../service/utils/ServiceRegistryUtils.java | 71 - .../yarn/service/utils/SliderFileSystem.java | 51 - .../hadoop/yarn/service/utils/SliderUtils.java | 1699 ------------------ .../yarn/service/utils/ZookeeperUtils.java | 146 -- .../src/main/proto/ClientAMProtocol.proto | 56 - .../hadoop/yarn/service/MockServiceAM.java | 221 --- .../hadoop/yarn/service/ServiceTestUtils.java | 59 - .../hadoop/yarn/service/TestServiceApiUtil.java | 529 ------ .../yarn/service/TestYarnNativeServices.java | 472 ----- .../client/TestBuildExternalComponents.java | 128 -- .../yarn/service/client/TestServiceCLI.java | 139 -- .../yarn/service/conf/ExampleAppJson.java | 65 - .../yarn/service/conf/TestAppJsonResolve.java | 224 --- .../service/conf/TestLoadExampleAppJson.java | 78 - .../service/conf/TestValidateServiceNames.java | 123 -- .../providers/TestAbstractClientProvider.java | 118 -- .../service/providers/TestProviderFactory.java | 76 - .../servicemonitor/TestServiceMonitor.java | 104 -- .../TestServiceTimelinePublisher.java | 293 --- .../src/test/resources/example-app.json | 15 - .../src/test/resources/log4j.properties | 66 - .../service/conf/examples/app-override.json | 72 - .../hadoop/yarn/service/conf/examples/app.json | 47 - .../yarn/service/conf/examples/default.json | 16 - .../yarn/service/conf/examples/external0.json | 8 - .../yarn/service/conf/examples/external1.json | 30 - .../yarn/service/conf/examples/external2.json | 22 - .../yarn/service/provider/docker/appConfig.json | 42 - .../yarn/service/provider/docker/resources.json | 16 - .../yarn/service/provider/docker/test.template | 16 - .../src/test/resources/yarn-site.xml | 19 - .../hadoop-yarn-slider/pom.xml | 39 - .../hadoop-yarn-applications/pom.xml | 2 +- .../native-services/NativeServicesDiscovery.md | 14 + .../native-services/NativeServicesIntro.md | 13 + hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +- 371 files changed, 23798 insertions(+), 24702 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 4483fa8..8aeeabd 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -87,19 +87,19 @@ </includes> </fileSet> <fileSet> - <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory> <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> <includes> <include>*-sources.jar</include> </includes> </fileSet> <fileSet> - <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory> <outputDirectory>etc/hadoop</outputDirectory> </fileSet> <fileSet> - <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory> - <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/lib/services</outputDirectory> </fileSet> <fileSet> <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml index 5de45a9..1b81f98 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-dist.xml @@ -17,7 +17,7 @@ <assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> - <id>hadoop-yarn-slider-dist</id> + <id>hadoop-yarn-services-dist</id> <formats> <format>dir</format> </formats> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e1fd707..b9eff92 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -427,7 +427,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-slider-core</artifactId> + <artifactId>hadoop-yarn-services-core</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 785f714..0a53455 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -49,7 +49,7 @@ function hadoop_usage hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" - hadoop_add_subcommand "servicesapi" "run yarn-native-service rest server" + hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "service" "run a service" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" @@ -151,14 +151,14 @@ function yarncmd_case scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; - servicesapi) + apiserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*' hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*' - HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApplicationApiWebApp' + HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ -${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\ ${HADOOP_HDFS_HOME}/${HDFS_DIR},\ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ @@ -171,11 +171,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; service) - hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services"'/*' HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ -${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services,\ ${HADOOP_HDFS_HOME}/${HDFS_DIR},\ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 02aec3b..90a87bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -159,4 +159,4 @@ # # See ResourceManager for some examples # -#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-servicesapi.log-$(date +'%Y%m%d%H%M')" +#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')" http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index fb2f2ba..7d9f15c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -91,7 +91,7 @@ <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-slider-core</artifactId> + <artifactId>hadoop-yarn-services-core</artifactId> <version>${project.version}</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java new file mode 100644 index 0000000..f55e3f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.webapp; + +import com.google.inject.Singleton; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.ServiceStatus; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; + +/** + * The rest API endpoints for users to manage services on YARN. + */ +@Singleton +@Path(CONTEXT_ROOT) +public class ApiServer { + private static final Logger LOG = + LoggerFactory.getLogger(ApiServer.class); + private static Configuration YARN_CONFIG = new YarnConfiguration(); + private static ServiceClient SERVICE_CLIENT; + + static { + init(); + } + + // initialize all the common resources - order is important + private static void init() { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + + @GET + @Path(VERSION) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) + public Response getVersion() { + String version = VersionInfo.getBuildVersion(); + LOG.info(version); + return Response.ok(version).build(); + } + + @POST + @Path(SERVICE_ROOT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response createService(Service service) { + LOG.info("POST: createService = {}", service); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service); + LOG.info("Successfully created service " + service.getName() + + " applicationId = " + applicationId); + serviceStatus.setState(ServiceState.ACCEPTED); + serviceStatus.setUri( + CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service + .getName()); + return Response.status(Status.CREATED).entity(serviceStatus).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.BAD_REQUEST).entity(serviceStatus) + .build(); + } catch (Exception e) { + String message = "Failed to create service " + service.getName(); + LOG.error(message, e); + serviceStatus.setDiagnostics(message + ": " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @GET + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getService(@PathParam(SERVICE_NAME) String appName) { + LOG.info("GET: getService for appName = {}", appName); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + Service app = SERVICE_CLIENT.getStatus(appName); + return Response.ok(app).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (Exception e) { + LOG.error("Get service failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @DELETE + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response deleteService(@PathParam(SERVICE_NAME) String appName) { + LOG.info("DELETE: deleteService for appName = {}", appName); + return stopService(appName, true); + } + + private Response stopService(String appName, boolean destroy) { + try { + SERVICE_CLIENT.actionStop(appName, destroy); + if (destroy) { + SERVICE_CLIENT.actionDestroy(appName); + LOG.info("Successfully deleted service {}", appName); + } else { + LOG.info("Successfully stopped service {}", appName); + } + return Response.status(Status.NO_CONTENT).build(); + } catch (ApplicationNotFoundException e) { + ServiceStatus serviceStatus = new ServiceStatus(); + serviceStatus.setDiagnostics( + "Service " + appName + " not found " + e.getMessage()); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (Exception e) { + ServiceStatus serviceStatus = new ServiceStatus(); + serviceStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @PUT + @Path(COMPONENT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) + public Response updateComponent(@PathParam(SERVICE_NAME) String appName, + @PathParam(COMPONENT_NAME) String componentName, Component component) { + + if (component.getNumberOfContainers() < 0) { + return Response.status(Status.BAD_REQUEST).entity( + "Service = " + appName + ", Component = " + component.getName() + + ": Invalid number of containers specified " + component + .getNumberOfContainers()).build(); + } + try { + Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName, + Collections.singletonMap(component.getName(), + component.getNumberOfContainers())); + return Response.ok().entity( + "Updating component " + componentName + " size from " + original + .get(componentName) + " to " + component.getNumberOfContainers()) + .build(); + } catch (YarnException | IOException e) { + ServiceStatus status = new ServiceStatus(); + status.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); + } + } + + @PUT + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateService(@PathParam(SERVICE_NAME) String appName, + Service updateServiceData) { + LOG.info("PUT: updateService for app = {} with data = {}", appName, + updateServiceData); + + // Ignore the app name provided in updateServiceData and always use appName + // path param + updateServiceData.setName(appName); + + // For STOP the app should be running. If already stopped then this + // operation will be a no-op. For START it should be in stopped state. + // If already running then this operation will be a no-op. + if (updateServiceData.getState() != null + && updateServiceData.getState() == ServiceState.STOPPED) { + return stopService(appName, false); + } + + // If a START is requested + if (updateServiceData.getState() != null + && updateServiceData.getState() == ServiceState.STARTED) { + return startService(appName); + } + + // If new lifetime value specified then update it + if (updateServiceData.getLifetime() != null + && updateServiceData.getLifetime() > 0) { + return updateLifetime(appName, updateServiceData); + } + + // flex a single component app + if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil + .hasComponent(updateServiceData)) { + Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData); + return updateComponent(updateServiceData.getName(), defaultComp.getName(), + defaultComp); + } + + // If nothing happens consider it a no-op + return Response.status(Status.NO_CONTENT).build(); + } + + private Response updateLifetime(String appName, Service updateAppData) { + try { + String newLifeTime = + SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + return Response.ok("Service " + appName + " lifeTime is successfully updated to " + + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); + } catch (Exception e) { + String message = + "Failed to update service (" + appName + ") lifetime (" + + updateAppData.getLifetime() + ")"; + LOG.error(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + " : " + e.getMessage()).build(); + } + } + + private Response startService(String appName) { + try { + SERVICE_CLIENT.actionStart(appName); + LOG.info("Successfully started service " + appName); + return Response.ok("Service " + appName + " is successfully started").build(); + } catch (Exception e) { + String message = "Failed to start service " + appName; + LOG.info(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + ": " + e.getMessage()).build(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java new file mode 100644 index 0000000..b226df7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.webapp; + +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.eclipse.jetty.webapp.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*; + +/** + * This class launches the web service using Hadoop HttpServer2 (which uses + * an embedded Jetty container). This is the entry point to your service. + * The Java command used to launch this app should call the main method. + */ +public class ApiServerWebApp extends AbstractService { + private static final Logger logger = LoggerFactory + .getLogger(ApiServerWebApp.class); + private static final String SEP = ";"; + + // REST API server for YARN native services + private HttpServer2 apiServer; + private InetSocketAddress bindAddress; + + public static void main(String[] args) throws IOException { + ApiServerWebApp apiWebApp = new ApiServerWebApp(); + try { + apiWebApp.startWebApp(); + } catch (Exception e) { + apiWebApp.close(); + } + } + + public ApiServerWebApp() { + super(ApiServerWebApp.class.getName()); + } + + @Override + protected void serviceStart() throws Exception { + bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS, + DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT); + logger.info("YARN API server running on " + bindAddress); + if (UserGroupInformation.isSecurityEnabled()) { + doSecureLogin(getConfig()); + } + startWebApp(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (apiServer != null) { + apiServer.stop(); + } + super.serviceStop(); + } + + private void doSecureLogin(org.apache.hadoop.conf.Configuration conf) + throws IOException { + SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB, + YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName()); + addFilters(conf); + } + + private void addFilters(org.apache.hadoop.conf.Configuration conf) { + // Always load pseudo authentication filter to parse "user.name" in an URL + // to identify a HTTP request's user. + boolean hasHadoopAuthFilterInitializer = false; + String filterInitializerConfKey = "hadoop.http.filter.initializers"; + Class<?>[] initializersClasses = + conf.getClasses(filterInitializerConfKey); + List<String> targets = new ArrayList<String>(); + if (initializersClasses != null) { + for (Class<?> initializer : initializersClasses) { + if (initializer.getName().equals( + AuthenticationFilterInitializer.class.getName())) { + hasHadoopAuthFilterInitializer = true; + break; + } + targets.add(initializer.getName()); + } + } + if (!hasHadoopAuthFilterInitializer) { + targets.add(AuthenticationFilterInitializer.class.getName()); + conf.set(filterInitializerConfKey, StringUtils.join(",", targets)); + } + } + + private void startWebApp() throws IOException { + URI uri = URI.create("http://"; + NetUtils.getHostPortString(bindAddress)); + + apiServer = new HttpServer2.Builder() + .setName("api-server") + .setConf(getConfig()) + .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) + .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY) + .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .addEndpoint(uri).build(); + + String apiPackages = + ApiServer.class.getPackage().getName() + SEP + + GenericExceptionHandler.class.getPackage().getName() + SEP + + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + apiServer.addJerseyResourcePackage(apiPackages, "/*"); + + try { + logger.info("Service starting up. Logging start..."); + apiServer.start(); + logger.info("Server status = {}", apiServer.toString()); + for (Configuration conf : apiServer.getWebAppContext() + .getConfigurations()) { + logger.info("Configurations = {}", conf); + } + logger.info("Context Path = {}", Collections.singletonList( + apiServer.getWebAppContext().getContextPath())); + logger.info("ResourceBase = {}", Collections.singletonList( + apiServer.getWebAppContext().getResourceBase())); + logger.info("War = {}", Collections + .singletonList(apiServer.getWebAppContext().getWar())); + } catch (Exception ex) { + logger.error("Hadoop HttpServer2 App **failed**", ex); + throw ex; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java deleted file mode 100644 index 30fc5ef..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiService.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.webapp; - -import com.google.inject.Singleton; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.ApplicationState; -import org.apache.hadoop.yarn.service.api.records.ApplicationStatus; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.utils.SliderUtils; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; - -@Singleton -@Path(CONTEXT_ROOT) -public class ApplicationApiService { - private static final Logger LOG = - LoggerFactory.getLogger(ApplicationApiService.class); - private static Configuration YARN_CONFIG = new YarnConfiguration(); - private static ServiceClient SERVICE_CLIENT; - - static { - init(); - } - - // initialize all the common resources - order is important - private static void init() { - SERVICE_CLIENT = new ServiceClient(); - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); - } - - @GET - @Path(VERSION) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) - public Response getVersion() { - String version = VersionInfo.getBuildVersion(); - LOG.info(version); - return Response.ok(version).build(); - } - - @POST - @Path(APP_ROOT_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response createApplication(Application application) { - LOG.info("POST: createApplication = {}", application); - ApplicationStatus applicationStatus = new ApplicationStatus(); - try { - ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application); - LOG.info("Successfully created application " + application.getName() - + " applicationId = " + applicationId); - applicationStatus.setState(ApplicationState.ACCEPTED); - applicationStatus.setUri( - CONTEXT_ROOT + APP_ROOT_PATH + "/" + application - .getName()); - return Response.status(Status.CREATED).entity(applicationStatus).build(); - } catch (IllegalArgumentException e) { - applicationStatus.setDiagnostics(e.getMessage()); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } catch (Exception e) { - String message = "Failed to create application " + application.getName(); - LOG.error(message, e); - applicationStatus.setDiagnostics(message + ": " + e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - - @GET - @Path(APP_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getApplication(@PathParam(APP_NAME) String appName) { - LOG.info("GET: getApplication for appName = {}", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - - // app name validation - if (!SliderUtils.isClusternameValid(appName)) { - applicationStatus.setDiagnostics("Invalid application name: " + appName); - applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } - - try { - Application app = SERVICE_CLIENT.getStatus(appName); - return Response.ok(app).build(); - } catch (Exception e) { - LOG.error("Get application failed", e); - applicationStatus - .setDiagnostics("Failed to retrieve application: " + e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - - @DELETE - @Path(APP_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response deleteApplication(@PathParam(APP_NAME) String appName) { - LOG.info("DELETE: deleteApplication for appName = {}", appName); - return stopApplication(appName, true); - } - - private Response stopApplication(String appName, boolean destroy) { - try { - SERVICE_CLIENT.actionStop(appName, destroy); - if (destroy) { - SERVICE_CLIENT.actionDestroy(appName); - LOG.info("Successfully deleted application {}", appName); - } else { - LOG.info("Successfully stopped application {}", appName); - } - return Response.status(Status.NO_CONTENT).build(); - } catch (ApplicationNotFoundException e) { - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics( - "Application " + appName + " not found " + e.getMessage()); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } catch (Exception e) { - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - - @PUT - @Path(COMPONENT_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) - public Response updateComponent(@PathParam(APP_NAME) String appName, - @PathParam(COMPONENT_NAME) String componentName, Component component) { - - if (component.getNumberOfContainers() < 0) { - return Response.status(Status.BAD_REQUEST).entity( - "Application = " + appName + ", Component = " + component.getName() - + ": Invalid number of containers specified " + component - .getNumberOfContainers()).build(); - } - try { - Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName, - Collections.singletonMap(component.getName(), - component.getNumberOfContainers())); - return Response.ok().entity( - "Updating component " + componentName + " size from " + original - .get(componentName) + " to " + component.getNumberOfContainers()) - .build(); - } catch (YarnException | IOException e) { - ApplicationStatus status = new ApplicationStatus(); - status.setDiagnostics(e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) - .build(); - } - } - - @PUT - @Path(APP_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response updateApplication(@PathParam(APP_NAME) String appName, - Application updateAppData) { - LOG.info("PUT: updateApplication for app = {} with data = {}", appName, - updateAppData); - - // Ignore the app name provided in updateAppData and always use appName - // path param - updateAppData.setName(appName); - - // For STOP the app should be running. If already stopped then this - // operation will be a no-op. For START it should be in stopped state. - // If already running then this operation will be a no-op. - if (updateAppData.getState() != null - && updateAppData.getState() == ApplicationState.STOPPED) { - return stopApplication(appName, false); - } - - // If a START is requested - if (updateAppData.getState() != null - && updateAppData.getState() == ApplicationState.STARTED) { - return startApplication(appName); - } - - // If new lifetime value specified then update it - if (updateAppData.getLifetime() != null - && updateAppData.getLifetime() > 0) { - return updateLifetime(appName, updateAppData); - } - - // flex a single component app - if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil - .hasComponent(updateAppData)) { - Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData); - return updateComponent(updateAppData.getName(), defaultComp.getName(), - defaultComp); - } - - // If nothing happens consider it a no-op - return Response.status(Status.NO_CONTENT).build(); - } - - private Response updateLifetime(String appName, Application updateAppData) { - try { - String newLifeTime = - SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); - return Response.ok("Application " + appName + " lifeTime is successfully updated to " - + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); - } catch (Exception e) { - String message = - "Failed to update application (" + appName + ") lifetime (" - + updateAppData.getLifetime() + ")"; - LOG.error(message, e); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(message + " : " + e.getMessage()).build(); - } - } - - private Response startApplication(String appName) { - try { - SERVICE_CLIENT.actionStart(appName); - LOG.info("Successfully started application " + appName); - return Response.ok("Application " + appName + " is successfully started").build(); - } catch (Exception e) { - String message = "Failed to start application " + appName; - LOG.info(message, e); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(message + ": " + e.getMessage()).build(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928a646/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java deleted file mode 100644 index 7225209..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApplicationApiWebApp.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.webapp; - -import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URI; -import java.util.Arrays; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.eclipse.jetty.webapp.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class launches the web application using Hadoop HttpServer2 (which uses - * an embedded Jetty container). This is the entry point to your application. - * The Java command used to launch this app should call the main method. - */ -public class ApplicationApiWebApp extends AbstractService { - private static final Logger logger = LoggerFactory - .getLogger(ApplicationApiWebApp.class); - private static final String SEP = ";"; - - // REST API server for YARN native services - private HttpServer2 applicationApiServer; - - public static void main(String[] args) throws IOException { - ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp(); - try { - apiWebApp.startWebApp(); - } catch (Exception e) { - if (apiWebApp != null) { - apiWebApp.close(); - } - } - } - - public ApplicationApiWebApp() { - super(ApplicationApiWebApp.class.getName()); - } - - @Override - protected void serviceStart() throws Exception { - startWebApp(); - super.serviceStart(); - } - - @Override - protected void serviceStop() throws Exception { - if (applicationApiServer != null) { - applicationApiServer.stop(); - } - super.serviceStop(); - } - - protected void startWebApp() throws IOException { - // The port that we should run on can be set into an environment variable - // Look for that variable and default to 9191 if it isn't there. - String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT); - if (StringUtils.isEmpty(webPort)) { - webPort = "9191"; - } - - String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST); - if (StringUtils.isEmpty(webHost)) { - webHost = InetAddress.getLocalHost().getHostName(); - } - logger.info("YARN native services REST API running on host {} and port {}", - webHost, webPort); - logger.info("Configuration = {}", getConfig()); - - applicationApiServer = new HttpServer2.Builder() - .setName("services-rest-api") - .addEndpoint(URI.create("http://"; + webHost + ":" + webPort)).build(); - - String apiPackages = - ApplicationApiService.class.getPackage().getName() + SEP - + GenericExceptionHandler.class.getPackage().getName() + SEP - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); - applicationApiServer.addJerseyResourcePackage(apiPackages, "/*"); - - try { - logger.info("Application starting up. Logging start..."); - applicationApiServer.start(); - logger.info("Server status = {}", applicationApiServer.toString()); - for (Configuration conf : applicationApiServer.getWebAppContext() - .getConfigurations()) { - logger.info("Configurations = {}", conf); - } - logger.info("Context Path = {}", Arrays.asList(applicationApiServer - .getWebAppContext().getContextPath())); - logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer - .getWebAppContext().getResourceBase())); - logger.info("War = {}", - Arrays.asList(applicationApiServer.getWebAppContext().getWar())); - } catch (Exception ex) { - logger.error("Hadoop HttpServer2 App **failed**", ex); - throw ex; - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
