YARN-5355: YARN Timeline Service v.2: alpha 2 (varunsaxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/32cba6c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/32cba6c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/32cba6c3 Branch: refs/heads/YARN-3926 Commit: 32cba6c3036dfb1dcb0a4fd77a68ddad17dd4082 Parents: 4cae120 3d00c8f Author: Varun Saxena <varunsax...@apache.org> Authored: Wed Aug 30 11:41:10 2017 +0530 Committer: Varun Saxena <varunsax...@apache.org> Committed: Wed Aug 30 11:41:10 2017 +0530 ---------------------------------------------------------------------- .../src/main/conf/hadoop-policy.xml | 11 + .../jobhistory/JobHistoryEventHandler.java | 27 +- .../v2/app/job/impl/TaskAttemptImpl.java | 58 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 19 +- .../v2/app/rm/RMContainerAllocator.java | 14 +- .../hadoop/mapreduce/jobhistory/TestEvents.java | 4 +- .../jobhistory/TestJobHistoryEventHandler.java | 8 +- .../v2/app/rm/TestRMContainerAllocator.java | 136 +++ .../jobhistory/MapAttemptFinishedEvent.java | 87 +- .../jobhistory/ReduceAttemptFinishedEvent.java | 83 +- .../jobhistory/TaskAttemptFinishedEvent.java | 47 +- .../TaskAttemptUnsuccessfulCompletionEvent.java | 48 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 51 +- .../mapreduce/jobhistory/TaskFinishedEvent.java | 42 +- .../mapred/TestMRTimelineEventHandling.java | 30 +- .../mapreduce/JobHistoryFileReplayMapperV1.java | 3 +- .../hadoop/mapreduce/SimpleEntityWriterV1.java | 3 +- hadoop-project/src/site/markdown/index.md.vm | 8 +- .../api/protocolrecords/AllocateResponse.java | 32 +- .../hadoop/yarn/api/records/CollectorInfo.java | 59 + .../timelineservice/ApplicationEntity.java | 28 + .../records/timelineservice/TimelineEntity.java | 52 +- .../hadoop/yarn/conf/YarnConfiguration.java | 75 +- .../hadoop/yarn/util/TimelineServiceHelper.java | 8 + .../src/main/proto/yarn_protos.proto | 5 + .../src/main/proto/yarn_service_protos.proto | 2 +- .../timelineservice/TestApplicationEntity.java | 71 ++ .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../distributedshell/ApplicationMaster.java | 42 +- .../distributedshell/TestDistributedShell.java | 173 +-- .../api/async/impl/AMRMClientAsyncImpl.java | 19 +- .../ApplicationMasterServiceProtoTestBase.java | 72 ++ .../hadoop/yarn/client/ProtocolHATestBase.java | 20 +- ...ationMasterServiceProtocolForTimelineV2.java | 71 ++ ...estApplicationMasterServiceProtocolOnHA.java | 46 +- .../api/async/impl/TestAMRMClientAsync.java | 2 +- .../impl/pb/AllocateResponsePBImpl.java | 37 +- .../records/impl/pb/CollectorInfoPBImpl.java | 152 +++ .../yarn/client/api/TimelineV2Client.java | 10 +- .../client/api/impl/TimelineV2ClientImpl.java | 117 +- .../src/main/resources/yarn-default.xml | 44 + .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../api/impl/TestTimelineClientV2Impl.java | 56 +- .../ApplicationHistoryServer.java | 79 +- .../security/TimelineAuthenticationFilter.java | 49 - ...TimelineAuthenticationFilterInitializer.java | 129 --- ...lineDelegationTokenSecretManagerService.java | 240 ----- ...neV1DelegationTokenSecretManagerService.java | 225 ++++ .../TestTimelineAuthenticationFilter.java | 323 ------ .../TestTimelineAuthenticationFilterForV1.java | 332 ++++++ ...TimelineAuthenticationFilterInitializer.java | 76 -- .../protocolrecords/NodeHeartbeatRequest.java | 13 +- .../protocolrecords/NodeHeartbeatResponse.java | 8 +- .../ReportNewCollectorInfoRequest.java | 13 +- .../impl/pb/NodeHeartbeatRequestPBImpl.java | 74 +- .../impl/pb/NodeHeartbeatResponsePBImpl.java | 61 +- .../pb/ReportNewCollectorInfoRequestPBImpl.java | 36 +- .../server/api/records/AppCollectorData.java | 125 +++ .../server/api/records/AppCollectorsMap.java | 46 - .../records/impl/pb/AppCollectorDataPBImpl.java | 227 ++++ .../records/impl/pb/AppCollectorsMapPBImpl.java | 152 --- .../api/records/impl/pb/package-info.java | 19 + .../security/TimelineAuthenticationFilter.java | 55 + ...TimelineAuthenticationFilterInitializer.java | 137 +++ ...elineDelgationTokenSecretManagerService.java | 83 ++ .../server/timeline/security/package-info.java | 26 + .../util/timeline/TimelineServerUtils.java | 92 ++ .../yarn/server/util/timeline/package-info.java | 25 + .../yarn_server_common_service_protos.proto | 16 +- .../java/org/apache/hadoop/yarn/TestRPC.java | 36 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 48 +- ...TimelineAuthenticationFilterInitializer.java | 78 ++ .../hadoop/yarn/server/nodemanager/Context.java | 14 +- .../yarn/server/nodemanager/NodeManager.java | 64 +- .../nodemanager/NodeStatusUpdaterImpl.java | 57 +- .../collectormanager/NMCollectorService.java | 43 +- .../containermanager/AuxServices.java | 3 +- .../containermanager/ContainerManagerImpl.java | 16 +- .../ApplicationContainerFinishedEvent.java | 9 +- .../application/ApplicationImpl.java | 33 +- .../containermanager/container/Container.java | 2 + .../container/ContainerImpl.java | 22 +- .../recovery/NMLeveldbStateStoreService.java | 21 +- .../recovery/NMNullStateStoreService.java | 2 +- .../recovery/NMStateStoreService.java | 13 +- .../security/authorize/NMPolicyProvider.java | 22 +- .../timelineservice/NMTimelinePublisher.java | 52 +- .../amrmproxy/BaseAMRMProxyTest.java | 9 +- .../application/TestApplication.java | 2 +- .../recovery/NMMemoryStateStoreService.java | 4 +- .../TestNMLeveldbStateStoreService.java | 6 +- .../nodemanager/webapp/MockContainer.java | 4 + .../nodemanager/webapp/TestNMWebServer.java | 4 +- .../ApplicationMasterService.java | 5 +- .../resourcemanager/DefaultAMSProcessor.java | 8 +- .../resourcemanager/ResourceTrackerService.java | 88 +- .../metrics/TimelineServiceV2Publisher.java | 12 +- .../server/resourcemanager/rmapp/RMApp.java | 35 +- .../resourcemanager/rmapp/RMAppEventType.java | 3 - .../server/resourcemanager/rmapp/RMAppImpl.java | 27 +- .../yarn/server/resourcemanager/MockNM.java | 16 + .../TestRMHATimelineCollectors.java | 126 +++ .../TestResourceTrackerService.java | 32 +- .../applicationsmanager/MockAsm.java | 17 +- .../TestSystemMetricsPublisherForV2.java | 13 +- .../server/resourcemanager/rmapp/MockRMApp.java | 18 +- .../hadoop-yarn-server-tests/pom.xml | 11 + .../hadoop/yarn/server/TestRMNMSecretKeys.java | 34 +- .../TestTimelineServiceClientIntegration.java | 15 +- .../security/TestTimelineAuthFilterForV2.java | 478 +++++++++ .../src/test/resources/krb5.conf | 28 - .../AbstractTimelineReaderHBaseTestBase.java | 177 +++ ...stTimelineReaderWebServicesHBaseStorage.java | 1007 ++++++++++++------ .../storage/DataGeneratorForTest.java | 423 +++++--- .../storage/TestHBaseTimelineStorageApps.java | 442 +++++--- .../TestHBaseTimelineStorageEntities.java | 544 +++++++--- .../storage/TestHBaseTimelineStorageSchema.java | 135 +++ .../storage/flow/TestFlowDataGenerator.java | 28 +- .../flow/TestHBaseStorageFlowActivity.java | 72 +- .../storage/flow/TestHBaseStorageFlowRun.java | 237 +++-- .../flow/TestHBaseStorageFlowRunCompaction.java | 67 +- .../reader/filter/TimelineFilterUtils.java | 17 + .../storage/HBaseTimelineReaderImpl.java | 12 +- .../storage/HBaseTimelineWriterImpl.java | 294 ++--- .../storage/TimelineSchemaCreator.java | 52 +- .../storage/application/ApplicationColumn.java | 48 - .../application/ApplicationColumnPrefix.java | 52 - .../storage/application/ApplicationRowKey.java | 49 +- .../storage/application/ApplicationTable.java | 2 +- .../storage/apptoflow/AppToFlowColumn.java | 47 - .../apptoflow/AppToFlowColumnPrefix.java | 206 ++++ .../storage/apptoflow/AppToFlowRowKey.java | 101 +- .../storage/apptoflow/AppToFlowTable.java | 21 +- .../storage/common/BaseTable.java | 37 +- .../storage/common/ColumnHelper.java | 50 +- .../common/HBaseTimelineStorageUtils.java | 124 ++- .../storage/common/KeyConverterToString.java | 38 + .../storage/common/LongConverter.java | 2 +- .../storage/entity/EntityColumn.java | 48 - .../storage/entity/EntityColumnPrefix.java | 51 - .../storage/entity/EntityRowKey.java | 100 +- .../storage/entity/EntityRowKeyPrefix.java | 11 +- .../storage/entity/EntityTable.java | 4 +- .../storage/flow/FlowActivityColumnPrefix.java | 58 +- .../storage/flow/FlowActivityRowKey.java | 59 +- .../storage/flow/FlowRunColumn.java | 53 +- .../storage/flow/FlowRunColumnPrefix.java | 53 +- .../storage/flow/FlowRunCoprocessor.java | 36 +- .../storage/flow/FlowRunRowKey.java | 47 +- .../storage/flow/FlowRunTable.java | 13 +- .../timelineservice/storage/package-info.java | 6 +- .../reader/AbstractTimelineStorageReader.java | 158 +++ .../storage/reader/ApplicationEntityReader.java | 77 +- .../storage/reader/EntityTypeReader.java | 179 ++++ .../reader/FlowActivityEntityReader.java | 30 +- .../storage/reader/FlowRunEntityReader.java | 53 +- .../storage/reader/GenericEntityReader.java | 201 ++-- .../reader/SubApplicationEntityReader.java | 488 +++++++++ .../storage/reader/TimelineEntityReader.java | 60 +- .../reader/TimelineEntityReaderFactory.java | 18 +- .../subapplication/SubApplicationColumn.java | 108 ++ .../SubApplicationColumnFamily.java | 68 ++ .../SubApplicationColumnPrefix.java | 250 +++++ .../subapplication/SubApplicationRowKey.java | 290 +++++ .../SubApplicationRowKeyPrefix.java | 69 ++ .../subapplication/SubApplicationTable.java | 174 +++ .../storage/subapplication/package-info.java | 28 + .../common/TestHBaseTimelineStorageUtils.java | 33 + .../storage/common/TestKeyConverters.java | 4 + .../storage/common/TestRowKeys.java | 54 +- .../storage/common/TestRowKeysAsString.java | 144 +++ .../collector/AppLevelTimelineCollector.java | 143 +-- .../AppLevelTimelineCollectorWithAgg.java | 150 +++ .../collector/NodeTimelineCollectorManager.java | 275 ++++- .../PerNodeTimelineCollectorsAuxService.java | 12 +- .../collector/TimelineCollector.java | 18 +- .../collector/TimelineCollectorManager.java | 14 +- .../reader/TimelineDataToRetrieve.java | 35 +- .../reader/TimelineEntityFilters.java | 160 +-- .../reader/TimelineReaderContext.java | 37 +- .../reader/TimelineReaderManager.java | 40 +- .../reader/TimelineReaderServer.java | 77 +- .../reader/TimelineReaderUtils.java | 31 +- .../reader/TimelineReaderWebServices.java | 693 ++++++++++-- .../reader/TimelineReaderWebServicesUtils.java | 98 +- .../reader/TimelineUIDConverter.java | 84 +- ...neReaderAuthenticationFilterInitializer.java | 53 + ...elineReaderWhitelistAuthorizationFilter.java | 123 +++ ...WhitelistAuthorizationFilterInitializer.java | 66 ++ .../reader/security/package-info.java | 25 + .../CollectorNodemanagerSecurityInfo.java | 69 ++ ...neV2DelegationTokenSecretManagerService.java | 126 +++ .../timelineservice/security/package-info.java | 25 + .../storage/FileSystemTimelineReaderImpl.java | 21 + .../storage/FileSystemTimelineWriterImpl.java | 15 +- .../timelineservice/storage/TimelineReader.java | 23 +- .../timelineservice/storage/TimelineWriter.java | 28 +- .../storage/common/TimelineStorageUtils.java | 1 - .../org.apache.hadoop.security.SecurityInfo | 14 + .../TestNMTimelineCollectorManager.java | 4 +- .../collector/TestTimelineCollector.java | 12 +- .../reader/TestTimelineReaderWebServices.java | 4 +- ...elineReaderWhitelistAuthorizationFilter.java | 380 +++++++ .../reader/TestTimelineUIDConverter.java | 17 +- .../TestFileSystemTimelineReaderImpl.java | 115 +- .../TestFileSystemTimelineWriterImpl.java | 8 +- .../src/site/markdown/TimelineServiceV2.md | 433 +++++++- 207 files changed, 12631 insertions(+), 4174 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org