YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db76a3ad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db76a3ad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db76a3ad Branch: refs/heads/YARN-2928 Commit: db76a3ad03682fdafcbec5276eef11ecdbd2719d Parents: 10a4f8a Author: Sangjin Lee <sj...@apache.org> Authored: Mon Feb 8 12:17:43 2016 -0800 Committer: Sangjin Lee <sj...@apache.org> Committed: Mon Feb 8 12:17:43 2016 -0800 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 170 ++++++++++--------- .../hadoop/mapreduce/jobhistory/TestEvents.java | 2 +- .../mapreduce/util/JobHistoryEventUtils.java | 7 +- .../hadoop/mapred/TimelineEntityConverter.java | 6 +- .../hadoop/mapreduce/JobHistoryFileParser.java | 3 + hadoop-yarn-project/CHANGES.txt | 3 + .../ApplicationAttemptEntity.java | 9 +- .../timelineservice/ApplicationEntity.java | 9 +- .../records/timelineservice/ClusterEntity.java | 6 +- .../timelineservice/ContainerEntity.java | 9 +- .../records/timelineservice/FlowRunEntity.java | 9 +- .../HierarchicalTimelineEntity.java | 8 +- .../records/timelineservice/QueueEntity.java | 6 +- .../timelineservice/TimelineEntities.java | 11 +- .../records/timelineservice/TimelineEntity.java | 106 ++++++------ .../timelineservice/TimelineEntityType.java | 71 +++++--- .../records/timelineservice/TimelineEvent.java | 30 ++-- .../records/timelineservice/TimelineMetric.java | 39 +++-- .../timelineservice/TimelineWriteResponse.java | 59 +++---- .../api/records/timelineservice/UserEntity.java | 6 +- .../records/timelineservice/package-info.java | 8 +- .../hadoop/yarn/conf/YarnConfiguration.java | 9 +- .../hadoop/yarn/client/api/TimelineClient.java | 13 +- .../client/api/impl/TimelineClientImpl.java | 16 +- .../yarn/util/timeline/TimelineUtils.java | 21 ++- .../yarn/server/nodemanager/NodeManager.java | 2 +- .../collectormanager/NMCollectorService.java | 9 +- .../collectormanager/package-info.java | 28 +++ .../timelineservice/NMTimelineEvent.java | 4 + .../timelineservice/NMTimelineEventType.java | 3 + .../timelineservice/NMTimelinePublisher.java | 14 +- .../timelineservice/package-info.java | 29 ++++ .../resourcemanager/RMActiveServiceContext.java | 8 +- .../server/resourcemanager/RMContextImpl.java | 4 +- .../metrics/AbstractSystemMetricsPublisher.java | 20 ++- .../metrics/NoOpSystemMetricPublisher.java | 2 +- .../metrics/SystemMetricsPublisher.java | 3 + .../metrics/TimelineServiceV1Publisher.java | 8 +- .../metrics/TimelineServiceV2Publisher.java | 7 +- .../resourcemanager/metrics/package-info.java | 28 +++ .../rmapp/RMAppCollectorUpdateEvent.java | 3 + .../server/resourcemanager/rmapp/RMAppImpl.java | 5 +- .../RMTimelineCollectorManager.java | 33 ++-- .../timelineservice/package-info.java | 28 +++ .../collector/AppLevelTimelineCollector.java | 3 +- .../collector/NodeTimelineCollectorManager.java | 8 +- .../PerNodeTimelineCollectorsAuxService.java | 2 + .../collector/TimelineCollector.java | 2 + .../collector/TimelineCollectorManager.java | 12 +- .../collector/TimelineCollectorWebService.java | 70 +++++--- .../timelineservice/collector/package-info.java | 29 ++++ .../reader/TimelineReaderManager.java | 32 +++- .../reader/TimelineReaderServer.java | 2 +- .../reader/TimelineReaderWebServices.java | 28 +-- .../reader/TimelineReaderWebServicesUtils.java | 50 +++--- .../reader/TimelineUIDConverter.java | 10 +- .../reader/filter/TimelineFilterUtils.java | 8 +- .../timelineservice/reader/package-info.java | 6 + .../storage/FileSystemTimelineReaderImpl.java | 70 ++++---- .../storage/FileSystemTimelineWriterImpl.java | 7 +- .../storage/HBaseTimelineReaderImpl.java | 3 + .../storage/HBaseTimelineWriterImpl.java | 29 ++-- .../storage/OfflineAggregationWriter.java | 13 +- .../PhoenixOfflineAggregationWriterImpl.java | 27 +-- .../storage/TimelineAggregationTrack.java | 2 +- .../timelineservice/storage/TimelineReader.java | 6 +- .../storage/TimelineSchemaCreator.java | 4 +- .../timelineservice/storage/TimelineWriter.java | 15 +- .../storage/application/ApplicationColumn.java | 4 +- .../application/ApplicationColumnPrefix.java | 8 +- .../storage/application/ApplicationRowKey.java | 33 ++-- .../storage/application/ApplicationTable.java | 16 +- .../storage/application/package-info.java | 4 + .../storage/apptoflow/AppToFlowColumn.java | 6 +- .../apptoflow/AppToFlowColumnFamily.java | 2 +- .../storage/apptoflow/AppToFlowRowKey.java | 9 +- .../storage/apptoflow/AppToFlowTable.java | 6 +- .../storage/apptoflow/package-info.java | 5 + .../storage/common/BaseTable.java | 21 ++- .../common/BufferedMutatorDelegator.java | 2 +- .../timelineservice/storage/common/Column.java | 16 +- .../storage/common/ColumnFamily.java | 2 +- .../storage/common/ColumnHelper.java | 24 +-- .../storage/common/ColumnPrefix.java | 22 ++- .../storage/common/LongConverter.java | 5 +- .../storage/common/NumericValueConverter.java | 7 +- .../storage/common/OfflineAggregationInfo.java | 61 ++++--- .../timelineservice/storage/common/Range.java | 3 + .../storage/common/Separator.java | 20 ++- .../common/TimelineHBaseSchemaConstants.java | 33 ++-- .../storage/common/TimelineStorageUtils.java | 79 +++++---- .../storage/common/TimestampGenerator.java | 12 +- .../storage/common/ValueConverter.java | 10 +- .../storage/entity/EntityColumn.java | 2 +- .../storage/entity/EntityColumnPrefix.java | 9 +- .../storage/entity/EntityRowKey.java | 54 +++--- .../storage/entity/EntityTable.java | 16 +- .../storage/entity/package-info.java | 4 + .../flow/AggregationCompactionDimension.java | 6 +- .../storage/flow/AggregationOperation.java | 10 +- .../storage/flow/FlowActivityColumnFamily.java | 3 +- .../storage/flow/FlowActivityColumnPrefix.java | 19 ++- .../storage/flow/FlowActivityRowKey.java | 34 ++-- .../storage/flow/FlowActivityTable.java | 19 ++- .../storage/flow/FlowRunColumnPrefix.java | 8 +- .../storage/flow/FlowRunCoprocessor.java | 8 +- .../storage/flow/FlowRunRowKey.java | 21 ++- .../storage/flow/FlowRunTable.java | 8 +- .../storage/flow/FlowScanner.java | 57 ++++--- .../storage/flow/package-info.java | 29 ++++ .../timelineservice/storage/package-info.java | 4 + .../storage/reader/ApplicationEntityReader.java | 28 +-- .../reader/FlowActivityEntityReader.java | 2 +- .../storage/reader/FlowRunEntityReader.java | 14 +- .../storage/reader/GenericEntityReader.java | 58 +++++-- .../storage/reader/TimelineEntityReader.java | 80 ++++++++- .../reader/TimelineEntityReaderFactory.java | 18 +- .../storage/reader/package-info.java | 5 + 118 files changed, 1420 insertions(+), 818 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 129ff5a..6e5afb1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -135,7 +135,8 @@ public class JobHistoryEventHandler extends AbstractService private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB"; private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK"; - private static String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE = "MAPREDUCE_TASK_ATTEMPT"; + private static final String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE = + "MAPREDUCE_TASK_ATTEMPT"; public JobHistoryEventHandler(AppContext context, int startCount) { super("JobHistoryEventHandler"); @@ -467,8 +468,9 @@ public class JobHistoryEventHandler extends AbstractService try { if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) { threadPool.shutdownNow(); - if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) + if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) { LOG.error("ThreadPool did not terminate"); + } } } catch (InterruptedException ie) { threadPool.shutdownNow(); @@ -1061,7 +1063,7 @@ public class JobHistoryEventHandler extends AbstractService default: break; } - + try { timelineClient.putEntities(tEntity); } catch (IOException|YarnException ex) { @@ -1070,12 +1072,12 @@ public class JobHistoryEventHandler extends AbstractService } } - private void putEntityWithoutBlocking(final TimelineClient timelineClient, + private void putEntityWithoutBlocking(final TimelineClient client, final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity) { Runnable publishWrapper = new Runnable() { public void run() { try { - timelineClient.putEntities(entity); + client.putEntities(entity); } catch (IOException|YarnException e) { LOG.error("putEntityNonBlocking get failed: " + e); throw new RuntimeException(e.toString()); @@ -1141,87 +1143,92 @@ public class JobHistoryEventHandler extends AbstractService entity.addIsRelatedToEntity(relatedTaskEntity, taskId); return entity; } - - private void processEventForNewTimelineService(HistoryEvent event, JobId jobId, - long timestamp) { - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null; + + private void processEventForNewTimelineService(HistoryEvent event, + JobId jobId, long timestamp) { + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = + null; String taskId = null; String taskAttemptId = null; boolean setCreatedTime = false; switch (event.getEventType()) { - // Handle job events - case JOB_SUBMITTED: - setCreatedTime = true; - break; - case JOB_STATUS_CHANGED: - case JOB_INFO_CHANGED: - case JOB_INITED: - case JOB_PRIORITY_CHANGED: - case JOB_QUEUE_CHANGED: - case JOB_FAILED: - case JOB_KILLED: - case JOB_ERROR: - case JOB_FINISHED: - case AM_STARTED: - case NORMALIZED_RESOURCE: - break; - // Handle task events - case TASK_STARTED: - setCreatedTime = true; - taskId = ((TaskStartedEvent)event).getTaskId().toString(); - break; - case TASK_FAILED: - taskId = ((TaskFailedEvent)event).getTaskId().toString(); - break; - case TASK_UPDATED: - taskId = ((TaskUpdatedEvent)event).getTaskId().toString(); - break; - case TASK_FINISHED: - taskId = ((TaskFinishedEvent)event).getTaskId().toString(); - break; - case MAP_ATTEMPT_STARTED: - case REDUCE_ATTEMPT_STARTED: - setCreatedTime = true; - taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); - taskAttemptId = ((TaskAttemptStartedEvent)event). - getTaskAttemptId().toString(); - break; - case CLEANUP_ATTEMPT_STARTED: - case SETUP_ATTEMPT_STARTED: - taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); - taskAttemptId = ((TaskAttemptStartedEvent)event). - getTaskAttemptId().toString(); - break; - case MAP_ATTEMPT_FAILED: - case CLEANUP_ATTEMPT_FAILED: - case REDUCE_ATTEMPT_FAILED: - case SETUP_ATTEMPT_FAILED: - case MAP_ATTEMPT_KILLED: - case CLEANUP_ATTEMPT_KILLED: - case REDUCE_ATTEMPT_KILLED: - case SETUP_ATTEMPT_KILLED: - taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event).getTaskId().toString(); - taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event). - getTaskAttemptId().toString(); - break; - case MAP_ATTEMPT_FINISHED: - taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString(); - taskAttemptId = ((MapAttemptFinishedEvent)event).getAttemptId().toString(); - break; - case REDUCE_ATTEMPT_FINISHED: - taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString(); - taskAttemptId = ((ReduceAttemptFinishedEvent)event).getAttemptId().toString(); - break; - case SETUP_ATTEMPT_FINISHED: - case CLEANUP_ATTEMPT_FINISHED: - taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString(); - taskAttemptId = ((TaskAttemptFinishedEvent)event).getAttemptId().toString(); - break; - default: - LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" + - " and handled by timeline service."); - return; + // Handle job events + case JOB_SUBMITTED: + setCreatedTime = true; + break; + case JOB_STATUS_CHANGED: + case JOB_INFO_CHANGED: + case JOB_INITED: + case JOB_PRIORITY_CHANGED: + case JOB_QUEUE_CHANGED: + case JOB_FAILED: + case JOB_KILLED: + case JOB_ERROR: + case JOB_FINISHED: + case AM_STARTED: + case NORMALIZED_RESOURCE: + break; + // Handle task events + case TASK_STARTED: + setCreatedTime = true; + taskId = ((TaskStartedEvent)event).getTaskId().toString(); + break; + case TASK_FAILED: + taskId = ((TaskFailedEvent)event).getTaskId().toString(); + break; + case TASK_UPDATED: + taskId = ((TaskUpdatedEvent)event).getTaskId().toString(); + break; + case TASK_FINISHED: + taskId = ((TaskFinishedEvent)event).getTaskId().toString(); + break; + case MAP_ATTEMPT_STARTED: + case REDUCE_ATTEMPT_STARTED: + setCreatedTime = true; + taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); + taskAttemptId = ((TaskAttemptStartedEvent)event). + getTaskAttemptId().toString(); + break; + case CLEANUP_ATTEMPT_STARTED: + case SETUP_ATTEMPT_STARTED: + taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); + taskAttemptId = ((TaskAttemptStartedEvent)event). + getTaskAttemptId().toString(); + break; + case MAP_ATTEMPT_FAILED: + case CLEANUP_ATTEMPT_FAILED: + case REDUCE_ATTEMPT_FAILED: + case SETUP_ATTEMPT_FAILED: + case MAP_ATTEMPT_KILLED: + case CLEANUP_ATTEMPT_KILLED: + case REDUCE_ATTEMPT_KILLED: + case SETUP_ATTEMPT_KILLED: + taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event). + getTaskId().toString(); + taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event). + getTaskAttemptId().toString(); + break; + case MAP_ATTEMPT_FINISHED: + taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString(); + taskAttemptId = ((MapAttemptFinishedEvent)event). + getAttemptId().toString(); + break; + case REDUCE_ATTEMPT_FINISHED: + taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString(); + taskAttemptId = ((ReduceAttemptFinishedEvent)event). + getAttemptId().toString(); + break; + case SETUP_ATTEMPT_FINISHED: + case CLEANUP_ATTEMPT_FINISHED: + taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString(); + taskAttemptId = ((TaskAttemptFinishedEvent)event). + getAttemptId().toString(); + break; + default: + LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" + + " and handled by timeline service."); + return; } if (taskId == null) { // JobEntity @@ -1240,7 +1247,6 @@ public class JobHistoryEventHandler extends AbstractService taskId, setCreatedTime); } } - putEntityWithoutBlocking(timelineClient, tEntity); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java index e7d5006..6eb8a43 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java @@ -405,7 +405,7 @@ public class TestEvents { public void setDatum(Object datum) { this.datum = datum; } - + @Override public TimelineEvent toTimelineEvent() { return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java index f4896ff..e60380b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java @@ -26,7 +26,12 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.node.ArrayNode; import org.codehaus.jackson.node.ObjectNode; -public class JobHistoryEventUtils { +/** + * Class containing utility methods to be used by JobHistoryEventHandler. + */ +public final class JobHistoryEventUtils { + private JobHistoryEventUtils() { + } public static JsonNode countersToJSON(Counters counters) { ObjectMapper mapper = new ObjectMapper(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java index 80928dc..880014b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java @@ -125,7 +125,8 @@ class TimelineEntityConverter { } } - private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) { + private Set<TimelineEntity> createTaskAndTaskAttemptEntities( + JobInfo jobInfo) { Set<TimelineEntity> entities = new HashSet<>(); Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks(); LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() + @@ -175,7 +176,8 @@ class TimelineEntityConverter { return taskAttempts; } - private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) { + private TimelineEntity createTaskAttemptEntity( + TaskAttemptInfo taskAttemptInfo) { TimelineEntity taskAttempt = new TimelineEntity(); taskAttempt.setType(TASK_ATTEMPT); taskAttempt.setId(taskAttemptInfo.getAttemptId().toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java index c290cd6..5d9dc0b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java @@ -28,6 +28,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; +/** + * Used to parse job history and configuration files. + */ class JobHistoryFileParser { private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ae5d907..f6bf667 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -229,6 +229,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-4644. TestRMRestart fails and findbugs issue in YARN-2928 branch. (Varun Saxena via Naganarasimha G R) + YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun + Saxena via sjlee) + Trunk - Unreleased INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java index 734c741..053d84e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents an application attempt. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationAttemptEntity extends HierarchicalTimelineEntity { @@ -29,8 +32,10 @@ public class ApplicationAttemptEntity extends HierarchicalTimelineEntity { public ApplicationAttemptEntity(TimelineEntity entity) { super(entity); - if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + if (!entity.getType().equals( + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 183d8d8..6075ec4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents an application. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationEntity extends HierarchicalTimelineEntity { @@ -32,8 +35,10 @@ public class ApplicationEntity extends HierarchicalTimelineEntity { public ApplicationEntity(TimelineEntity entity) { super(entity); - if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + if (!entity.getType().equals( + TimelineEntityType.YARN_APPLICATION.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java index 94eefa8..1f96505 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents a YARN cluster. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class ClusterEntity extends HierarchicalTimelineEntity { @@ -30,7 +33,8 @@ public class ClusterEntity extends HierarchicalTimelineEntity { public ClusterEntity(TimelineEntity entity) { super(entity); if (!entity.getType().equals(TimelineEntityType.YARN_CLUSTER.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java index c7d6bce..f61920f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents a container belonging to an application attempt. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class ContainerEntity extends HierarchicalTimelineEntity { @@ -29,8 +32,10 @@ public class ContainerEntity extends HierarchicalTimelineEntity { public ContainerEntity(TimelineEntity entity) { super(entity); - if (!entity.getType().equals(TimelineEntityType.YARN_CONTAINER.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + if (!entity.getType().equals( + TimelineEntityType.YARN_CONTAINER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java index 3c3ffb4..410a1bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java @@ -22,6 +22,9 @@ import javax.xml.bind.annotation.XmlElement; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents a flow run. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class FlowRunEntity extends HierarchicalTimelineEntity { @@ -44,8 +47,10 @@ public class FlowRunEntity extends HierarchicalTimelineEntity { public FlowRunEntity(TimelineEntity entity) { super(entity); - if (!entity.getType().equals(TimelineEntityType.YARN_FLOW_RUN.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + if (!entity.getType().equals( + TimelineEntityType.YARN_FLOW_RUN.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } // set config to null setConfigs(null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java index 6235da0..4744e39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java @@ -17,17 +17,18 @@ */ package org.apache.hadoop.yarn.api.records.timelineservice; -import com.google.common.base.Joiner; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; +/** + * This class extends timeline entity and defines parent-child relationships + * with other entities. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public abstract class HierarchicalTimelineEntity extends TimelineEntity { @@ -66,6 +67,7 @@ public abstract class HierarchicalTimelineEntity extends TimelineEntity { setParent(new Identifier(type, id)); } + @SuppressWarnings("unchecked") public Set<Identifier> getChildren() { Object identifiers = getInfo().get(CHILDREN_INFO_KEY); if (identifiers == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java index d1a1f19..b654450 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents a queue. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class QueueEntity extends HierarchicalTimelineEntity { @@ -30,7 +33,8 @@ public class QueueEntity extends HierarchicalTimelineEntity { public QueueEntity(TimelineEntity entity) { super(entity); if (!entity.getType().equals(TimelineEntityType.YARN_QUEUE.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java index 39504cc..f08a0ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java @@ -27,6 +27,9 @@ import javax.xml.bind.annotation.XmlRootElement; import java.util.HashSet; import java.util.Set; +/** + * This class hosts a set of timeline entities. + */ @XmlRootElement(name = "entities") @XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @@ -44,12 +47,12 @@ public class TimelineEntities { return entities; } - public void setEntities(Set<TimelineEntity> entities) { - this.entities = entities; + public void setEntities(Set<TimelineEntity> timelineEntities) { + this.entities = timelineEntities; } - public void addEntities(Set<TimelineEntity> entities) { - this.entities.addAll(entities); + public void addEntities(Set<TimelineEntity> timelineEntities) { + this.entities.addAll(timelineEntities); } public void addEntity(TimelineEntity entity) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index a661f7a..acc132e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -54,6 +54,9 @@ import java.util.TreeSet; public class TimelineEntity implements Comparable<TimelineEntity> { protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_"; + /** + * Identifier of timeline entity(entity id + entity type). + */ @XmlRootElement(name = "identifier") @XmlAccessorType(XmlAccessType.NONE) public static class Identifier { @@ -74,8 +77,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> { return type; } - public void setType(String type) { - this.type = type; + public void setType(String entityType) { + this.type = entityType; } @XmlElement(name = "id") @@ -83,8 +86,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> { return id; } - public void setId(String id) { - this.id = id; + public void setId(String entityId) { + this.id = entityId; } @Override @@ -106,8 +109,9 @@ public class TimelineEntity implements Comparable<TimelineEntity> { @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; + } if (!(obj instanceof Identifier)) { return false; } @@ -208,11 +212,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void setIdentifier(Identifier identifier) { + public void setIdentifier(Identifier entityIdentifier) { if (real == null) { - this.identifier = identifier; + this.identifier = entityIdentifier; } else { - real.setIdentifier(identifier); + real.setIdentifier(entityIdentifier); } } @@ -235,19 +239,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void setInfo(Map<String, Object> info) { + public void setInfo(Map<String, Object> entityInfos) { if (real == null) { - this.info = TimelineServiceHelper.mapCastToHashMap(info); + this.info = TimelineServiceHelper.mapCastToHashMap(entityInfos); } else { - real.setInfo(info); + real.setInfo(entityInfos); } } - public void addInfo(Map<String, Object> info) { + public void addInfo(Map<String, Object> entityInfos) { if (real == null) { - this.info.putAll(info); + this.info.putAll(entityInfos); } else { - real.addInfo(info); + real.addInfo(entityInfos); } } @@ -278,19 +282,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void setConfigs(Map<String, String> configs) { + public void setConfigs(Map<String, String> entityConfigs) { if (real == null) { - this.configs = TimelineServiceHelper.mapCastToHashMap(configs); + this.configs = TimelineServiceHelper.mapCastToHashMap(entityConfigs); } else { - real.setConfigs(configs); + real.setConfigs(entityConfigs); } } - public void addConfigs(Map<String, String> configs) { + public void addConfigs(Map<String, String> entityConfigs) { if (real == null) { - this.configs.putAll(configs); + this.configs.putAll(entityConfigs); } else { - real.addConfigs(configs); + real.addConfigs(entityConfigs); } } @@ -311,19 +315,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void setMetrics(Set<TimelineMetric> metrics) { + public void setMetrics(Set<TimelineMetric> entityMetrics) { if (real == null) { - this.metrics = metrics; + this.metrics = entityMetrics; } else { - real.setMetrics(metrics); + real.setMetrics(entityMetrics); } } - public void addMetrics(Set<TimelineMetric> metrics) { + public void addMetrics(Set<TimelineMetric> entityMetrics) { if (real == null) { - this.metrics.addAll(metrics); + this.metrics.addAll(entityMetrics); } else { - real.addMetrics(metrics); + real.addMetrics(entityMetrics); } } @@ -344,19 +348,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void setEvents(NavigableSet<TimelineEvent> events) { + public void setEvents(NavigableSet<TimelineEvent> entityEvents) { if (real == null) { - this.events = events; + this.events = entityEvents; } else { - real.setEvents(events); + real.setEvents(entityEvents); } } - public void addEvents(Set<TimelineEvent> events) { + public void addEvents(Set<TimelineEvent> entityEvents) { if (real == null) { - this.events.addAll(events); + this.events.addAll(entityEvents); } else { - real.addEvents(events); + real.addEvents(entityEvents); } } @@ -389,20 +393,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> { @JsonSetter("isrelatedto") public void setIsRelatedToEntities( - Map<String, Set<String>> isRelatedToEntities) { + Map<String, Set<String>> isRelatedTo) { if (real == null) { this.isRelatedToEntities = - TimelineServiceHelper.mapCastToHashMap(isRelatedToEntities); + TimelineServiceHelper.mapCastToHashMap(isRelatedTo); } else { - real.setIsRelatedToEntities(isRelatedToEntities); + real.setIsRelatedToEntities(isRelatedTo); } } public void addIsRelatedToEntities( - Map<String, Set<String>> isRelatedToEntities) { + Map<String, Set<String>> isRelatedTo) { if (real == null) { - for (Map.Entry<String, Set<String>> entry : isRelatedToEntities - .entrySet()) { + for (Map.Entry<String, Set<String>> entry : isRelatedTo.entrySet()) { Set<String> ids = this.isRelatedToEntities.get(entry.getKey()); if (ids == null) { ids = new HashSet<>(); @@ -411,7 +414,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> { ids.addAll(entry.getValue()); } } else { - real.addIsRelatedToEntities(isRelatedToEntities); + real.addIsRelatedToEntities(isRelatedTo); } } @@ -447,10 +450,9 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } } - public void addRelatesToEntities(Map<String, Set<String>> relatesToEntities) { + public void addRelatesToEntities(Map<String, Set<String>> relatesTo) { if (real == null) { - for (Map.Entry<String, Set<String>> entry : relatesToEntities - .entrySet()) { + for (Map.Entry<String, Set<String>> entry : relatesTo.entrySet()) { Set<String> ids = this.relatesToEntities.get(entry.getKey()); if (ids == null) { ids = new HashSet<>(); @@ -459,7 +461,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> { ids.addAll(entry.getValue()); } } else { - real.addRelatesToEntities(relatesToEntities); + real.addRelatesToEntities(relatesTo); } } @@ -477,12 +479,12 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } @JsonSetter("relatesto") - public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) { + public void setRelatesToEntities(Map<String, Set<String>> relatesTo) { if (real == null) { this.relatesToEntities = - TimelineServiceHelper.mapCastToHashMap(relatesToEntities); + TimelineServiceHelper.mapCastToHashMap(relatesTo); } else { - real.setRelatesToEntities(relatesToEntities); + real.setRelatesToEntities(relatesTo); } } @@ -496,11 +498,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } @JsonSetter("createdtime") - public void setCreatedTime(long createdTime) { + public void setCreatedTime(long createdTs) { if (real == null) { - this.createdTime = createdTime; + this.createdTime = createdTs; } else { - real.setCreatedTime(createdTime); + real.setCreatedTime(createdTs); } } @@ -530,10 +532,12 @@ public class TimelineEntity implements Comparable<TimelineEntity> { @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (!(obj instanceof TimelineEntity)) + } + if (!(obj instanceof TimelineEntity)) { return false; + } TimelineEntity other = (TimelineEntity) obj; return getIdentifier().equals(other.getIdentifier()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java index ba32e20..8fcc2ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * Defines type of entity. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public enum TimelineEntityType { @@ -34,51 +37,63 @@ public enum TimelineEntityType { /** * Whether the input type can be a parent of this entity. + * + * @param type entity type. + * @return true, if this entity type is parent of passed entity type, false + * otherwise. */ public boolean isParent(TimelineEntityType type) { switch (this) { - case YARN_CLUSTER: - return false; - case YARN_FLOW_RUN: - return YARN_FLOW_RUN == type || YARN_CLUSTER == type; - case YARN_APPLICATION: - return YARN_FLOW_RUN == type || YARN_CLUSTER == type; - case YARN_APPLICATION_ATTEMPT: - return YARN_APPLICATION == type; - case YARN_CONTAINER: - return YARN_APPLICATION_ATTEMPT == type; - case YARN_QUEUE: - return YARN_QUEUE == type; - default: - return false; + case YARN_CLUSTER: + return false; + case YARN_FLOW_RUN: + return YARN_FLOW_RUN == type || YARN_CLUSTER == type; + case YARN_APPLICATION: + return YARN_FLOW_RUN == type || YARN_CLUSTER == type; + case YARN_APPLICATION_ATTEMPT: + return YARN_APPLICATION == type; + case YARN_CONTAINER: + return YARN_APPLICATION_ATTEMPT == type; + case YARN_QUEUE: + return YARN_QUEUE == type; + default: + return false; } } /** * Whether the input type can be a child of this entity. + * + * @param type entity type. + * @return true, if this entity type is child of passed entity type, false + * otherwise. */ public boolean isChild(TimelineEntityType type) { switch (this) { - case YARN_CLUSTER: - return YARN_FLOW_RUN == type || YARN_APPLICATION == type; - case YARN_FLOW_RUN: - return YARN_FLOW_RUN == type || YARN_APPLICATION == type; - case YARN_APPLICATION: - return YARN_APPLICATION_ATTEMPT == type; - case YARN_APPLICATION_ATTEMPT: - return YARN_CONTAINER == type; - case YARN_CONTAINER: - return false; - case YARN_QUEUE: - return YARN_QUEUE == type; - default: - return false; + case YARN_CLUSTER: + return YARN_FLOW_RUN == type || YARN_APPLICATION == type; + case YARN_FLOW_RUN: + return YARN_FLOW_RUN == type || YARN_APPLICATION == type; + case YARN_APPLICATION: + return YARN_APPLICATION_ATTEMPT == type; + case YARN_APPLICATION_ATTEMPT: + return YARN_CONTAINER == type; + case YARN_CONTAINER: + return false; + case YARN_QUEUE: + return YARN_QUEUE == type; + default: + return false; } } /** * Whether the type of this entity matches the type indicated by the input * argument. + * + * @param typeString entity type represented as a string. + * @return true, if string representation of this entity type matches the + * entity type passed. */ public boolean matches(String typeString) { return toString().equals(typeString); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java index a563658..87fc291 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java @@ -28,6 +28,11 @@ import javax.xml.bind.annotation.XmlRootElement; import java.util.HashMap; import java.util.Map; +/** + * This class contains the information of an event that belongs to an entity. + * Users are free to define what the event means, such as starting an + * application, container being allocated, etc. + */ @XmlRootElement(name = "event") @XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @@ -48,8 +53,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> { return id; } - public void setId(String id) { - this.id = id; + public void setId(String eventId) { + this.id = eventId; } // required by JAXB @@ -63,12 +68,12 @@ public class TimelineEvent implements Comparable<TimelineEvent> { return info; } - public void setInfo(Map<String, Object> info) { - this.info = TimelineServiceHelper.mapCastToHashMap(info); + public void setInfo(Map<String, Object> infos) { + this.info = TimelineServiceHelper.mapCastToHashMap(infos); } - public void addInfo(Map<String, Object> info) { - this.info.putAll(info); + public void addInfo(Map<String, Object> infos) { + this.info.putAll(infos); } public void addInfo(String key, Object value) { @@ -80,8 +85,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> { return timestamp; } - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; + public void setTimestamp(long ts) { + this.timestamp = ts; } public boolean isValid() { @@ -97,15 +102,18 @@ public class TimelineEvent implements Comparable<TimelineEvent> { @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (!(o instanceof TimelineEvent)) + } + if (!(o instanceof TimelineEvent)) { return false; + } TimelineEvent event = (TimelineEvent) o; - if (timestamp != event.timestamp) + if (timestamp != event.timestamp) { return false; + } if (!id.equals(event.id)) { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java index e3870da..2f60515 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java @@ -28,12 +28,19 @@ import java.util.Comparator; import java.util.Map; import java.util.TreeMap; +/** + * This class contains the information of a metric that is related to some + * entity. Metric can either be a time series or single value. + */ @XmlRootElement(name = "metric") @XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class TimelineMetric { + /** + * Type of metric. + */ public static enum Type { SINGLE_VALUE, TIME_SERIES @@ -63,8 +70,8 @@ public class TimelineMetric { return type; } - public void setType(Type type) { - this.type = type; + public void setType(Type metricType) { + this.type = metricType; } @XmlElement(name = "id") @@ -72,8 +79,8 @@ public class TimelineMetric { return id; } - public void setId(String id) { - this.id = id; + public void setId(String metricId) { + this.id = metricId; } // required by JAXB @@ -87,24 +94,24 @@ public class TimelineMetric { return values; } - public void setValues(Map<Long, Number> values) { + public void setValues(Map<Long, Number> vals) { if (type == Type.SINGLE_VALUE) { - overwrite(values); + overwrite(vals); } else { if (values != null) { this.values = new TreeMap<Long, Number>(reverseComparator); - this.values.putAll(values); + this.values.putAll(vals); } else { this.values = null; } } } - public void addValues(Map<Long, Number> values) { + public void addValues(Map<Long, Number> vals) { if (type == Type.SINGLE_VALUE) { - overwrite(values); + overwrite(vals); } else { - this.values.putAll(values); + this.values.putAll(vals); } } @@ -115,14 +122,14 @@ public class TimelineMetric { values.put(timestamp, value); } - private void overwrite(Map<Long, Number> values) { - if (values.size() > 1) { + private void overwrite(Map<Long, Number> vals) { + if (vals.size() > 1) { throw new IllegalArgumentException( "Values cannot contain more than one point in " + Type.SINGLE_VALUE + " mode"); } this.values.clear(); - this.values.putAll(values); + this.values.putAll(vals); } public boolean isValid() { @@ -139,10 +146,12 @@ public class TimelineMetric { // Only check if type and id are equal @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (!(o instanceof TimelineMetric)) + } + if (!(o instanceof TimelineMetric)) { return false; + } TimelineMetric m = (TimelineMetric) o; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java index 4739d8f..eda1ee2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java @@ -29,9 +29,9 @@ import java.util.List; /** * A class that holds a list of put errors. This is the response returned when a - * list of {@link TimelineEntity} objects is added to the timeline. If there are errors - * in storing individual entity objects, they will be indicated in the list of - * errors. + * list of {@link TimelineEntity} objects is added to the timeline. If there are + * errors in storing individual entity objects, they will be indicated in the + * list of errors. */ @XmlRootElement(name = "response") @XmlAccessorType(XmlAccessType.NONE) @@ -46,7 +46,7 @@ public class TimelineWriteResponse { } /** - * Get a list of {@link TimelineWriteError} instances + * Get a list of {@link TimelineWriteError} instances. * * @return a list of {@link TimelineWriteError} instances */ @@ -56,7 +56,7 @@ public class TimelineWriteResponse { } /** - * Add a single {@link TimelineWriteError} instance into the existing list + * Add a single {@link TimelineWriteError} instance into the existing list. * * @param error * a single {@link TimelineWriteError} instance @@ -66,24 +66,24 @@ public class TimelineWriteResponse { } /** - * Add a list of {@link TimelineWriteError} instances into the existing list + * Add a list of {@link TimelineWriteError} instances into the existing list. * - * @param errors + * @param writeErrors * a list of {@link TimelineWriteError} instances */ - public void addErrors(List<TimelineWriteError> errors) { - this.errors.addAll(errors); + public void addErrors(List<TimelineWriteError> writeErrors) { + this.errors.addAll(writeErrors); } /** - * Set the list to the given list of {@link TimelineWriteError} instances + * Set the list to the given list of {@link TimelineWriteError} instances. * - * @param errors + * @param writeErrors * a list of {@link TimelineWriteError} instances */ - public void setErrors(List<TimelineWriteError> errors) { + public void setErrors(List<TimelineWriteError> writeErrors) { this.errors.clear(); - this.errors.addAll(errors); + this.errors.addAll(writeErrors); } /** @@ -106,7 +106,7 @@ public class TimelineWriteResponse { private int errorCode; /** - * Get the entity Id + * Get the entity Id. * * @return the entity Id */ @@ -116,17 +116,16 @@ public class TimelineWriteResponse { } /** - * Set the entity Id + * Set the entity Id. * - * @param entityId - * the entity Id + * @param id the entity Id. */ - public void setEntityId(String entityId) { - this.entityId = entityId; + public void setEntityId(String id) { + this.entityId = id; } /** - * Get the entity type + * Get the entity type. * * @return the entity type */ @@ -136,17 +135,16 @@ public class TimelineWriteResponse { } /** - * Set the entity type + * Set the entity type. * - * @param entityType - * the entity type + * @param type the entity type. */ - public void setEntityType(String entityType) { - this.entityType = entityType; + public void setEntityType(String type) { + this.entityType = type; } /** - * Get the error code + * Get the error code. * * @return an error code */ @@ -156,13 +154,12 @@ public class TimelineWriteResponse { } /** - * Set the error code to the given error code + * Set the error code to the given error code. * - * @param errorCode - * an error code + * @param code an error code. */ - public void setErrorCode(int errorCode) { - this.errorCode = errorCode; + public void setErrorCode(int code) { + this.errorCode = code; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java index a229fd8..ced57c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * This entity represents a user. + */ @InterfaceAudience.Public @InterfaceStability.Unstable public class UserEntity extends TimelineEntity { @@ -30,7 +33,8 @@ public class UserEntity extends TimelineEntity { public UserEntity(TimelineEntity entity) { super(entity); if (!entity.getType().equals(TimelineEntityType.YARN_USER.toString())) { - throw new IllegalArgumentException("Incompatible entity type: " + getId()); + throw new IllegalArgumentException("Incompatible entity type: " + + getId()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java index 89a9e9b..c43bd62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java @@ -15,7 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@InterfaceAudience.Public package org.apache.hadoop.yarn.api.records.timelineservice; + +/** + * Package org.apache.hadoop.yarn.api.records.timelineservice contains classes + * which define the data model for ATSv2. + */ +@InterfaceAudience.Public +package org.apache.hadoop.yarn.api.records.timelineservice; import org.apache.hadoop.classification.InterfaceAudience; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6a12d8f..9b43fbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -401,7 +401,8 @@ public class YarnConfiguration extends Configuration { */ public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = RM_PREFIX + "system-metrics-publisher.enabled"; - public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false; + public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = + false; /** * The setting that controls whether yarn system metrics is published on the @@ -423,8 +424,8 @@ public class YarnConfiguration extends Configuration { public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size"; - public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = - 10; + public static final int + DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10; //RM delegation token related keys public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = @@ -1638,7 +1639,7 @@ public class YarnConfiguration extends Configuration { public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX = TIMELINE_SERVICE_PREFIX + "ui-web-path."; - /** Timeline client settings */ + /** Timeline client settings. */ public static final String TIMELINE_SERVICE_CLIENT_PREFIX = TIMELINE_SERVICE_PREFIX + "client."; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index 59ac5b3..ade4f9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -53,7 +53,7 @@ public abstract class TimelineClient extends AbstractService { * construct and initialize a timeline client if the following operations are * supposed to be conducted by that user. */ - protected ApplicationId contextAppId; + private ApplicationId contextAppId; /** * Creates an instance of the timeline v.1.x client. @@ -76,7 +76,7 @@ public abstract class TimelineClient extends AbstractService { @Private protected TimelineClient(String name, ApplicationId appId) { super(name); - contextAppId = appId; + setContextAppId(appId); } /** @@ -240,11 +240,18 @@ public abstract class TimelineClient extends AbstractService { /** * <p> - * Update the timeline service address where the request will be sent to + * Update the timeline service address where the request will be sent to. * </p> * @param address * the timeline service address */ public abstract void setTimelineServiceAddress(String address); + protected ApplicationId getContextAppId() { + return contextAppId; + } + + protected void setContextAppId(ApplicationId appId) { + this.contextAppId = appId; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 3e63242..a158a56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -390,8 +390,8 @@ public class TimelineClientImpl extends TimelineClient { entitiesContainer.addEntity(entity); } MultivaluedMap<String, String> params = new MultivaluedMapImpl(); - if (contextAppId != null) { - params.add("appid", contextAppId.toString()); + if (getContextAppId() != null) { + params.add("appid", getContextAppId().toString()); } if (async) { params.add("async", Boolean.TRUE.toString()); @@ -429,8 +429,7 @@ public class TimelineClientImpl extends TimelineClient { URI uri = constructResURI(getConfig(), timelineServiceAddress, true); putObjects(uri, path, params, obj); needRetry = false; - } - catch (Exception e) { + } catch (Exception e) { // TODO only handle exception for timelineServiceAddress being updated. // skip retry for other exceptions. checkRetryWithSleep(retries, e); @@ -453,10 +452,9 @@ public class TimelineClientImpl extends TimelineClient { Thread.currentThread().interrupt(); } } else { - LOG.error( - "TimelineClient has reached to max retry times :" + - this.maxServiceRetries + " for service address: " + - timelineServiceAddress); + LOG.error("TimelineClient has reached to max retry times :" + + this.maxServiceRetries + " for service address: " + + timelineServiceAddress); if (e instanceof YarnException) { throw (YarnException)e; } else if (e instanceof IOException) { @@ -634,7 +632,7 @@ public class TimelineClientImpl extends TimelineClient { } /** - * Poll TimelineServiceAddress for maximum of retries times if it is null + * Poll TimelineServiceAddress for maximum of retries times if it is null. * @param retries * @return the left retry times */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index 64aa7d4..53f5af2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -45,8 +45,10 @@ import org.codehaus.jackson.map.ObjectMapper; public class TimelineUtils { public static final String FLOW_NAME_TAG_PREFIX = "TIMELINE_FLOW_NAME_TAG"; - public static final String FLOW_VERSION_TAG_PREFIX = "TIMELINE_FLOW_VERSION_TAG"; - public static final String FLOW_RUN_ID_TAG_PREFIX = "TIMELINE_FLOW_RUN_ID_TAG"; + public static final String FLOW_VERSION_TAG_PREFIX = + "TIMELINE_FLOW_VERSION_TAG"; + public static final String FLOW_RUN_ID_TAG_PREFIX = + "TIMELINE_FLOW_RUN_ID_TAG"; private static ObjectMapper mapper; @@ -125,38 +127,39 @@ public class TimelineUtils { return SecurityUtil.buildTokenService(timelineServiceAddr); } - public static String generateDefaultFlowNameBasedOnAppId(ApplicationId appId) { + public static String generateDefaultFlowNameBasedOnAppId( + ApplicationId appId) { return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId(); } /** - * Generate flow name tag + * Generate flow name tag. * * @param flowName flow name that identifies a distinct flow application which * can be run repeatedly over time - * @return + * @return flow name tag. */ public static String generateFlowNameTag(String flowName) { return FLOW_NAME_TAG_PREFIX + ":" + flowName; } /** - * Generate flow version tag + * Generate flow version tag. * * @param flowVersion flow version that keeps track of the changes made to the * flow - * @return + * @return flow version tag. */ public static String generateFlowVersionTag(String flowVersion) { return FLOW_VERSION_TAG_PREFIX + ":" + flowVersion; } /** - * Generate flow run ID tag + * Generate flow run ID tag. * * @param flowRunId flow run ID that identifies one instance (or specific * execution) of that flow - * @return + * @return flow run id tag. */ public static String generateFlowRunIdTag(long flowRunId) { return FLOW_RUN_ID_TAG_PREFIX + ":" + flowRunId; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 596b5d3..912003a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -457,7 +457,7 @@ public class NodeManager extends CompositeService protected final ConcurrentMap<ContainerId, Container> containers = new ConcurrentSkipListMap<ContainerId, Container>(); - protected Map<ApplicationId, String> registeredCollectors; + private Map<ApplicationId, String> registeredCollectors; protected final ConcurrentMap<ContainerId, org.apache.hadoop.yarn.api.records.Container> increasedContainers = http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java index 3ba81ce..548c861 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java @@ -52,12 +52,11 @@ public class NMCollectorService extends CompositeService implements private static final Log LOG = LogFactory.getLog(NMCollectorService.class); - final Context context; + private final Context context; private Server server; public NMCollectorService(Context context) { - super(NMCollectorService.class.getName()); this.context = context; } @@ -123,7 +122,8 @@ public class NMCollectorService extends CompositeService implements client.setTimelineServiceAddress(collectorAddr); } } - ((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap); + ((NodeManager.NMContext)context).addRegisteredCollectors( + newCollectorsMap); } return ReportNewCollectorInfoResponse.newInstance(); @@ -139,6 +139,7 @@ public class NMCollectorService extends CompositeService implements " doesn't exist on NM."); } return GetTimelineCollectorContextResponse.newInstance( - app.getUser(), app.getFlowName(), app.getFlowVersion(), app.getFlowRunId()); + app.getUser(), app.getFlowName(), app.getFlowVersion(), + app.getFlowRunId()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java new file mode 100644 index 0000000..7bf597b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.nodemanager.collectormanager contains + * classes for handling timeline collector information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.nodemanager.collectormanager; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java index af8d94c..f275b37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java @@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice; import org.apache.hadoop.yarn.event.AbstractEvent; +/** + * Event posted to NMTimelinePublisher which in turn publishes it to + * timelineservice v2. + */ public class NMTimelineEvent extends AbstractEvent<NMTimelineEventType> { public NMTimelineEvent(NMTimelineEventType type) { super(type); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java index c1129af..b4ae45a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice; +/** + * Type of {@link NMTimelineEvent}. + */ public enum NMTimelineEventType { // Publish the NM Timeline entity TIMELINE_ENTITY_PUBLISH, http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index affaaae..684feaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -127,7 +127,8 @@ public class NMTimelinePublisher extends CompositeService { memoryMetric.addValue(currentTimeMillis, pmemUsage); entity.addMetric(memoryMetric); } - if (cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE) { + if (cpuUsageTotalCoresPercentage != + ResourceCalculatorProcessTree.UNAVAILABLE) { TimelineMetric cpuMetric = new TimelineMetric(); cpuMetric.setId(ContainerMetric.CPU.toString() + pId); cpuMetric.addValue(currentTimeMillis, cpuUsageTotalCoresPercentage); @@ -189,7 +190,8 @@ public class NMTimelinePublisher extends CompositeService { putEntity(entity, containerId.getApplicationAttemptId().getApplicationId()); } - private static ContainerEntity createContainerEntity(ContainerId containerId) { + private static ContainerEntity createContainerEntity( + ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); Identifier parentIdentifier = new Identifier(); @@ -214,6 +216,7 @@ public class NMTimelinePublisher extends CompositeService { } } + @SuppressWarnings("unchecked") public void publishApplicationEvent(ApplicationEvent event) { // publish only when the desired event is received switch (event.getType()) { @@ -226,14 +229,14 @@ public class NMTimelinePublisher extends CompositeService { default: if (LOG.isDebugEnabled()) { - LOG.debug(event.getType() - + " is not a desired ApplicationEvent which needs to be published by" - + " NMTimelinePublisher"); + LOG.debug(event.getType() + " is not a desired ApplicationEvent which" + + " needs to be published by NMTimelinePublisher"); } break; } } + @SuppressWarnings("unchecked") public void publishContainerEvent(ContainerEvent event) { // publish only when the desired event is received switch (event.getType()) { @@ -251,6 +254,7 @@ public class NMTimelinePublisher extends CompositeService { } } + @SuppressWarnings("unchecked") public void publishLocalizationEvent(LocalizationEvent event) { // publish only when the desired event is received switch (event.getType()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java new file mode 100644 index 0000000..66233fd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.nodemanager.timelineservice contains + * classes related to publishing container events and other NM lifecycle events + * to ATSv2. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.nodemanager.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability;
