http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index fc05310..12daa95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -388,15 +388,15 @@ public class RMActiveServiceContext { @Private @Unstable public void setRMTimelineCollectorManager( - RMTimelineCollectorManager timelineCollectorManager) { - this.timelineCollectorManager = timelineCollectorManager; + RMTimelineCollectorManager collectorManager) { + this.timelineCollectorManager = collectorManager; } @Private @Unstable public void setSystemMetricsPublisher( - SystemMetricsPublisher systemMetricsPublisher) { - this.systemMetricsPublisher = systemMetricsPublisher; + SystemMetricsPublisher metricsPublisher) { + this.systemMetricsPublisher = metricsPublisher; } @Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index e122ab4..4c72912 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -379,8 +379,8 @@ public class RMContextImpl implements RMContext { @Override public void setSystemMetricsPublisher( - SystemMetricsPublisher systemMetricsPublisher) { - this.systemMetricsPublisher = systemMetricsPublisher; + SystemMetricsPublisher metricsPublisher) { + this.systemMetricsPublisher = metricsPublisher; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java index a8c00a4..d4a4fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java @@ -30,6 +30,10 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +/** + * Abstract implementation of SystemMetricsPublisher which is then extended by + * metrics publisher implementations depending on timeline service version. + */ public abstract class AbstractSystemMetricsPublisher extends CompositeService implements SystemMetricsPublisher { private MultiThreadedDispatcher dispatcher; @@ -46,13 +50,18 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService protected void serviceInit(Configuration conf) throws Exception { dispatcher = new MultiThreadedDispatcher(getConfig().getInt( - YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, - YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE)); + YarnConfiguration. + RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, + YarnConfiguration. + DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE)); dispatcher.setDrainEventsOnStop(); addIfService(dispatcher); super.serviceInit(conf); } + /** + * Dispatches ATS related events using multiple threads. + */ @SuppressWarnings({ "rawtypes", "unchecked" }) public static class MultiThreadedDispatcher extends CompositeService implements Dispatcher { @@ -107,7 +116,7 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService } /** - * EventType which is used while publishing the events + * EventType which is used while publishing the events. */ protected static enum SystemMetricsEventType { PUBLISH_ENTITY, PUBLISH_APPLICATION_FINISHED_ENTITY @@ -158,9 +167,10 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService if (other.getType() != null) { return false; } - } else + } else { if (!appId.equals(other.appId) || !getType().equals(other.getType())) { - return false; + return false; + } } return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java index 1810df1..c0c066a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; /** * This class does nothing when any of the methods are invoked on - * SystemMetricsPublisher + * SystemMetricsPublisher. */ public class NoOpSystemMetricPublisher implements SystemMetricsPublisher{ http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index f895bba..8e8acbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +/** + * Interface used to publish app/container events to timelineservice. + */ public interface SystemMetricsPublisher { void appCreated(RMApp app, long createdTime); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index ddc8a16..7dc5073 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -43,6 +43,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +/** + * This class is responsible for posting application, appattempt & Container + * lifecycle related events to timeline service v1. + */ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { private static final Log LOG = @@ -137,8 +141,8 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { entity.addEvent(tEvent); // sync sending of finish event to avoid possibility of saving application - // finished state in RMStateStore save without publishing in ATS - putEntity(entity);// sync event so that ATS update is done without fail + // finished state in RMStateStore save without publishing in ATS. + putEntity(entity); // sync event so that ATS update is done without fail. } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 1b3aa6f..14073d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -60,14 +60,14 @@ import com.google.common.annotations.VisibleForTesting; /** * This class is responsible for posting application, appattempt & Container - * lifecycle related events to timeline service V2 + * lifecycle related events to timeline service v2. */ @Private @Unstable public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { private static final Log LOG = LogFactory.getLog(TimelineServiceV2Publisher.class); - protected RMTimelineCollectorManager rmTimelineCollectorManager; + private RMTimelineCollectorManager rmTimelineCollectorManager; private boolean publishContainerMetrics; public TimelineServiceV2Publisher(RMContext rmContext) { @@ -342,7 +342,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { } } - private static ContainerEntity createContainerEntity(ContainerId containerId) { + private static ContainerEntity createContainerEntity( + ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java new file mode 100644 index 0000000..a8a3804 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.resourcemanager.metrics contains + * classes related to publishing app/container events to ATS. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.metrics; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java index 698c9b5..9642911 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import org.apache.hadoop.yarn.api.records.ApplicationId; +/** + * Event used for updating collector address in RMApp on node heartbeat. + */ public class RMAppCollectorUpdateEvent extends RMAppEvent { private final String appCollectorAddr; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 1ee6247..62b13ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -97,7 +97,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector; -import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.state.InvalidStateTransitionException; @@ -613,8 +612,8 @@ public class RMAppImpl implements RMApp, Recoverable { } @Override - public void setCollectorAddr(String collectorAddr) { - this.collectorAddr = collectorAddr; + public void setCollectorAddr(String collectorAddress) { + this.collectorAddr = collectorAddress; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java index 116bf64..ff055a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java @@ -28,7 +28,10 @@ import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; - +/** + * This class extends TimelineCollectorManager to provide RM specific + * implementations. + */ @InterfaceAudience.Private @InterfaceStability.Unstable public class RMTimelineCollectorManager extends TimelineCollectorManager { @@ -44,8 +47,8 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager { RMApp app = rmContext.getRMApps().get(appId); if (app == null) { throw new YarnRuntimeException( - "Unable to get the timeline collector context info for a non-existing app " + - appId); + "Unable to get the timeline collector context info for a " + + "non-existing app " + appId); } String userId = app.getUser(); if (userId != null && !userId.isEmpty()) { @@ -57,18 +60,18 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager { continue; } switch (parts[0].toUpperCase()) { - case TimelineUtils.FLOW_NAME_TAG_PREFIX: - collector.getTimelineEntityContext().setFlowName(parts[1]); - break; - case TimelineUtils.FLOW_VERSION_TAG_PREFIX: - collector.getTimelineEntityContext().setFlowVersion(parts[1]); - break; - case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX: - collector.getTimelineEntityContext().setFlowRunId( - Long.parseLong(parts[1])); - break; - default: - break; + case TimelineUtils.FLOW_NAME_TAG_PREFIX: + collector.getTimelineEntityContext().setFlowName(parts[1]); + break; + case TimelineUtils.FLOW_VERSION_TAG_PREFIX: + collector.getTimelineEntityContext().setFlowVersion(parts[1]); + break; + case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX: + collector.getTimelineEntityContext().setFlowRunId( + Long.parseLong(parts[1])); + break; + default: + break; } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java new file mode 100644 index 0000000..c470011 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.resourcemanager.timelineservice + * contains classes related to handling of app level collectors. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java index b148491..36dd7b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java @@ -55,7 +55,8 @@ public class AppLevelTimelineCollector extends TimelineCollector { // Current user usually is not the app user, but keep this field non-null context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName()); // Use app ID to generate a default flow name for orphan app - context.setFlowName(TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId)); + context.setFlowName( + TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId)); // Set the flow version to string 1 if it's an orphan app context.setFlowVersion("1"); // Set the flow run ID to 1 if it's an orphan app http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java index 0977ed9..785fb19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java @@ -48,13 +48,17 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; +/** + * Class on the NodeManager side that manages adding and removing collectors and + * their lifecycle. Also instantiates the per-node collector webapp. + */ @Private @Unstable public class NodeTimelineCollectorManager extends TimelineCollectorManager { private static final Log LOG = LogFactory.getLog(NodeTimelineCollectorManager.class); - // REST server for this collector manager + // REST server for this collector manager. private HttpServer2 timelineRestServer; private String timelineRestServerBindAddress; @@ -97,7 +101,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { } /** - * Launch the REST web server for this collector manager + * Launch the REST web server for this collector manager. */ private void startWebApp() { Configuration conf = getConfig(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java index b738530..041e7c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java @@ -108,6 +108,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { * The collector is also initialized and started. If the service already * exists, no new service is created. * + * @param appId Application Id to be added. * @return whether it was added successfully */ public boolean addApplication(ApplicationId appId) { @@ -122,6 +123,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { * collector is also stopped as a result. If the collector does not exist, no * change is made. * + * @param appId Application Id to be removed. * @return whether it was removed successfully */ public boolean removeApplication(ApplicationId appId) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java index bb7db12..15187d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java @@ -80,6 +80,8 @@ public abstract class TimelineCollector extends CompositeService { * @param entities entities to post * @param callerUgi the caller UGI * @return the response that contains the result of the post. + * @throws IOException if there is any exception encountered while putting + * entities. */ public TimelineWriteResponse putEntities(TimelineEntities entities, UserGroupInformation callerUgi) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java index 165754d..8f74ffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java @@ -68,8 +68,10 @@ public class TimelineCollectorManager extends AbstractService { // basis writerFlusher = Executors.newSingleThreadScheduledExecutor(); flushInterval = conf.getInt( - YarnConfiguration.TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS); + YarnConfiguration. + TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS, + YarnConfiguration. + DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS); super.serviceInit(conf); } @@ -102,6 +104,8 @@ public class TimelineCollectorManager extends AbstractService { * Put the collector into the collection if an collector mapped by id does * not exist. * + * @param appId Application Id for which collector needs to be put. + * @param collector timeline collector to be put. * @throws YarnRuntimeException if there was any exception in initializing * and starting the app level service * @return the collector associated with id after the potential put. @@ -140,6 +144,7 @@ public class TimelineCollectorManager extends AbstractService { * Removes the collector for the specified id. The collector is also stopped * as a result. If the collector does not exist, no change is made. * + * @param appId Application Id to remove. * @return whether it was removed successfully */ public boolean remove(ApplicationId appId) { @@ -162,6 +167,7 @@ public class TimelineCollectorManager extends AbstractService { /** * Returns the collector for the specified id. * + * @param appId Application Id for which we need to get the collector. * @return the collector or null if it does not exist */ public TimelineCollector get(ApplicationId appId) { @@ -171,6 +177,8 @@ public class TimelineCollectorManager extends AbstractService { /** * Returns whether the collector for the specified id exists in this * collection. + * @param appId Application Id. + * @return true if collector for the app id is found, false otherwise. */ public boolean containsTimelineCollector(ApplicationId appId) { return collectors.containsKey(appId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java index 8f595e2..2dff937 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java @@ -74,6 +74,9 @@ public class TimelineCollectorWebService { private @Context ServletContext context; + /** + * Gives information about timeline collector. + */ @XmlRootElement(name = "about") @XmlAccessorType(XmlAccessType.NONE) @Public @@ -86,8 +89,8 @@ public class TimelineCollectorWebService { } - public AboutInfo(String about) { - this.about = about; + public AboutInfo(String abt) { + this.about = abt; } @XmlElement(name = "About") @@ -95,14 +98,18 @@ public class TimelineCollectorWebService { return about; } - public void setAbout(String about) { - this.about = about; + public void setAbout(String abt) { + this.about = abt; } } /** * Return the description of the timeline web services. + * + * @param req Servlet request. + * @param res Servlet response. + * @return description of timeline web service. */ @GET @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) @@ -117,6 +124,15 @@ public class TimelineCollectorWebService { * Accepts writes to the collector, and returns a response. It simply routes * the request to the app level collector. It expects an application as a * context. + * + * @param req Servlet request. + * @param res Servlet response. + * @param async flag indicating whether its an async put or not. "true" + * indicates, its an async call. If null, its considered false. + * @param appId Application Id to which the entities to be put belong to. If + * appId is not there or it cannot be parsed, HTTP 400 will be sent back. + * @param entities timeline entities to be put. + * @return a Response with appropriate HTTP status. */ @PUT @Path("/entities") @@ -202,29 +218,29 @@ public class TimelineCollectorWebService { } if (type != null) { switch (type) { - case YARN_CLUSTER: - entitiesToReturn.addEntity(new ClusterEntity(entity)); - break; - case YARN_FLOW_RUN: - entitiesToReturn.addEntity(new FlowRunEntity(entity)); - break; - case YARN_APPLICATION: - entitiesToReturn.addEntity(new ApplicationEntity(entity)); - break; - case YARN_APPLICATION_ATTEMPT: - entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity)); - break; - case YARN_CONTAINER: - entitiesToReturn.addEntity(new ContainerEntity(entity)); - break; - case YARN_QUEUE: - entitiesToReturn.addEntity(new QueueEntity(entity)); - break; - case YARN_USER: - entitiesToReturn.addEntity(new UserEntity(entity)); - break; - default: - break; + case YARN_CLUSTER: + entitiesToReturn.addEntity(new ClusterEntity(entity)); + break; + case YARN_FLOW_RUN: + entitiesToReturn.addEntity(new FlowRunEntity(entity)); + break; + case YARN_APPLICATION: + entitiesToReturn.addEntity(new ApplicationEntity(entity)); + break; + case YARN_APPLICATION_ATTEMPT: + entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity)); + break; + case YARN_CONTAINER: + entitiesToReturn.addEntity(new ContainerEntity(entity)); + break; + case YARN_QUEUE: + entitiesToReturn.addEntity(new QueueEntity(entity)); + break; + case YARN_USER: + entitiesToReturn.addEntity(new UserEntity(entity)); + break; + default: + break; } } else { entitiesToReturn.addEntity(entity); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java new file mode 100644 index 0000000..1f7dd23 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.timelineservice.collector contains + * classes which can be used across collector. This package contains classes + * which are not related to storage implementations though. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.timelineservice.collector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index 84a97ea..4cff3bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -34,6 +34,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader; import com.google.common.annotations.VisibleForTesting; +/** + * This class wraps over the timeline reader store implementation. It does some + * non trivial manipulation of the timeline data before or after getting + * it from the backend store. + */ @Private @Unstable public class TimelineReaderManager extends AbstractService { @@ -114,9 +119,19 @@ public class TimelineReaderManager extends AbstractService { } /** - * Get a set of entities matching given predicates. The meaning of each - * argument has been documented with {@link TimelineReader#getEntities}. + * Get a set of entities matching given predicates by making a call to + * backend storage implementation. The meaning of each argument has been + * documented in detail with {@link TimelineReader#getEntities}.If cluster ID + * has not been supplied by the client, fills the cluster id from config + * before making a call to backend storage. After fetching entities from + * backend, fills the appropriate UID based on entity type for each entity. * + * @param context Timeline context within the scope of which entities have to + * be fetched. + * @param filters Filters which limit the number of entities to be returned. + * @param dataToRetrieve Data to carry in each entity fetched. + * @return a set of <cite>TimelineEntity</cite> objects. + * @throws IOException if any problem occurs while getting entities. * @see TimelineReader#getEntities */ public Set<TimelineEntity> getEntities(TimelineReaderContext context, @@ -135,9 +150,18 @@ public class TimelineReaderManager extends AbstractService { } /** - * Get single timeline entity. The meaning of each argument has been - * documented with {@link TimelineReader#getEntity}. + * Get single timeline entity by making a call to backend storage + * implementation. The meaning of each argument in detail has been + * documented with {@link TimelineReader#getEntity}. If cluster ID has not + * been supplied by the client, fills the cluster id from config before making + * a call to backend storage. After fetching entity from backend, fills the + * appropriate UID based on entity type. * + * @param context Timeline context within the scope of which entity has to be + * fetched. + * @param dataToRetrieve Data to carry in the entity fetched. + * @return A <cite>TimelineEntity</cite> object if found, null otherwise. + * @throws IOException if any problem occurs while getting entity. * @see TimelineReader#getEntity */ public TimelineEntity getEntity(TimelineReaderContext context, http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java index 6b4213d..97725e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java @@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; -/** Main class for Timeline Reader */ +/** Main class for Timeline Reader. */ @Private @Unstable public class TimelineReaderServer extends CompositeService { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index d68199a..0b9549b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -73,24 +73,25 @@ public class TimelineReaderWebServices { private static final String DATE_PATTERN = "yyyyMMdd"; @VisibleForTesting - static ThreadLocal<DateFormat> DATE_FORMAT = new ThreadLocal<DateFormat>() { - @Override - protected DateFormat initialValue() { - SimpleDateFormat format = - new SimpleDateFormat(DATE_PATTERN, Locale.ENGLISH); - format.setTimeZone(TimeZone.getTimeZone("GMT")); - format.setLenient(false); - return format; - } - }; + static final ThreadLocal<DateFormat> DATE_FORMAT = + new ThreadLocal<DateFormat>() { + @Override + protected DateFormat initialValue() { + SimpleDateFormat format = + new SimpleDateFormat(DATE_PATTERN, Locale.ENGLISH); + format.setTimeZone(TimeZone.getTimeZone("GMT")); + format.setLenient(false); + return format; + } + }; private void init(HttpServletResponse response) { response.setContentType(null); } - private static class DateRange { - Long dateStart; - Long dateEnd; + private static final class DateRange { + private Long dateStart; + private Long dateEnd; private DateRange(Long start, Long end) { this.dateStart = start; this.dateEnd = end; @@ -212,6 +213,7 @@ public class TimelineReaderWebServices { * @param uId a delimited string containing clusterid, userid, flow name, * flowrun id and app id which are extracted from UID and then used to * query backend(Mandatory path param). + * @param entityType Type of entities(Mandatory path param). * @param limit Number of entities to return(Optional query param). * @param createdTimeStart If specified, matched entities should not be * created before this timestamp(Optional query param). http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java index fc07e51..d12f7e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java @@ -44,15 +44,15 @@ final class TimelineReaderWebServicesUtils { /** * Parse the passed context information represented as strings and convert * into a {@link TimelineReaderContext} object. - * @param clusterId - * @param userId - * @param flowName - * @param flowRunId - * @param appId - * @param entityType - * @param entityId + * @param clusterId Cluster Id. + * @param userId User Id. + * @param flowName Flow Name. + * @param flowRunId Run id for the flow. + * @param appId App Id. + * @param entityType Entity Type. + * @param entityId Entity Id. * @return a {@link TimelineReaderContext} object. - * @throws Exception + * @throws Exception if any problem occurs during parsing. */ static TimelineReaderContext createTimelineReaderContext(String clusterId, String userId, String flowName, String flowRunId, String appId, @@ -65,17 +65,17 @@ final class TimelineReaderWebServicesUtils { /** * Parse the passed filters represented as strings and convert them into a * {@link TimelineEntityFilters} object. - * @param limit - * @param createdTimeStart - * @param createdTimeEnd - * @param relatesTo - * @param isRelatedTo - * @param infofilters - * @param conffilters - * @param metricfilters - * @param eventfilters + * @param limit Limit to number of entities to return. + * @param createdTimeStart Created time start for the entities to return. + * @param createdTimeEnd Created time end for the entities to return. + * @param relatesTo Entities to return must match relatesTo. + * @param isRelatedTo Entities to return must match isRelatedTo. + * @param infofilters Entities to return must match these info filters. + * @param conffilters Entities to return must match these metric filters. + * @param metricfilters Entities to return must match these metric filters. + * @param eventfilters Entities to return must match these event filters. * @return a {@link TimelineEntityFilters} object. - * @throws Exception + * @throws Exception if any problem occurs during parsing. */ static TimelineEntityFilters createTimelineEntityFilters(String limit, String createdTimeStart, String createdTimeEnd, String relatesTo, @@ -94,11 +94,11 @@ final class TimelineReaderWebServicesUtils { /** * Parse the passed fields represented as strings and convert them into a * {@link TimelineDataToRetrieve} object. - * @param confs - * @param metrics - * @param fields + * @param confs confs to retrieve. + * @param metrics metrics to retrieve. + * @param fields fields to retrieve. * @return a {@link TimelineDataToRetrieve} object. - * @throws Exception + * @throws Exception if any problem occurs during parsing. */ static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs, String metrics, String fields) throws Exception { @@ -192,7 +192,7 @@ final class TimelineReaderWebServicesUtils { * should be represented as "key1:value1,key2:value2,key3:value3". * @param str delimited string represented as key-value pairs. * @param pairsDelim key-value pairs are delimited by this delimiter. - * @param keyValuesDelim key and value are delimited by this delimiter. + * @param keyValDelim key and value are delimited by this delimiter. * @return a map of key-value pairs with both key and value being strings. */ static Map<String, String> parseKeyStrValueStr(String str, @@ -212,8 +212,8 @@ final class TimelineReaderWebServicesUtils { * should be represented as "key1:value1,key2:value2,key3:value3". * @param str delimited string represented as key-value pairs. * @param pairsDelim key-value pairs are delimited by this delimiter. - * @param keyValuesDelim key and value are delimited by this delimiter. - * @return a map of key-value pairs with key being a string and value amy + * @param keyValDelim key and value are delimited by this delimiter. + * @return a map of key-value pairs with key being a string and value, any * object. */ static Map<String, Object> parseKeyStrValueObj(String str, http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java index cac3f06..08e5405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java @@ -204,7 +204,7 @@ enum TimelineUIDConverter { /** * Split UID using {@link #UID_DELIMITER_CHAR} and {@link #UID_ESCAPE_CHAR}. - * @param uid + * @param uid UID to be splitted. * @return a list of different parts of UID split across delimiter. * @throws IllegalArgumentException if UID is not properly escaped. */ @@ -229,17 +229,19 @@ enum TimelineUIDConverter { /** * Encodes UID depending on UID implementation. - * @param context + * + * @param context Reader context. * @return UID represented as a string. */ abstract String encodeUID(TimelineReaderContext context); /** * Decodes UID depending on UID implementation. - * @param uId + * + * @param uId UID to be decoded. * @return a {@link TimelineReaderContext} object if UID passed can be * decoded, null otherwise. - * @throws Exception + * @throws Exception if any problem occurs while decoding. */ abstract TimelineReaderContext decodeUID(String uId) throws Exception; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java index da3c383..f902500 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java @@ -93,9 +93,11 @@ public final class TimelineFilterUtils { * Creates equivalent HBase {@link FilterList} from {@link TimelineFilterList} * while converting different timeline filters(of type {@link TimelineFilter}) * into their equivalent HBase filters. - * @param colPrefix - * @param filterList - * @return a {@link FilterList} object + * + * @param <T> Describes the type of column prefix. + * @param colPrefix column prefix which will be used for conversion. + * @param filterList timeline filter list which has to be converted. + * @return A {@link FilterList} object. */ public static <T> FilterList createHBaseFilterList(ColumnPrefix<T> colPrefix, TimelineFilterList filterList) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java index 51247bd..116509a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java @@ -15,6 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/** + * Package org.apache.hadoop.yarn.server.timelineservice.reader contains classes + * which can be used across reader. This package contains classes which are + * not related to storage implementations. + */ @InterfaceAudience.Private @InterfaceStability.Unstable package org.apache.hadoop.yarn.server.timelineservice.reader; http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index aa0071f..97e05dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -105,15 +105,16 @@ public class FileSystemTimelineReaderImpl extends AbstractService /** * Deserialize a POJO object from a JSON string. - * @param clazz - * class to be desirialized * - * @param jsonString - * json string to deserialize - * @return TimelineEntity object - * @throws IOException - * @throws JsonMappingException - * @throws JsonGenerationException + * @param <T> Describes the type of class to be returned. + * @param clazz class to be deserialized. + * @param jsonString JSON string to deserialize. + * @return An object based on class type. Used typically for + * <cite>TimelineEntity</cite> object. + * @throws IOException if the underlying input source has problems during + * parsing. + * @throws JsonMappingException if parser has problems parsing content. + * @throws JsonGenerationException if there is a problem in JSON writing. */ public static <T> T getTimelineRecordFromJSON( String jsonString, Class<T> clazz) @@ -128,33 +129,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService } for (Field field : fields) { switch(field) { - case CONFIGS: - finalEntity.setConfigs(real.getConfigs()); - break; - case METRICS: - finalEntity.setMetrics(real.getMetrics()); - break; - case INFO: - finalEntity.setInfo(real.getInfo()); - break; - case IS_RELATED_TO: - finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities()); - break; - case RELATES_TO: - finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities()); - break; - case EVENTS: - finalEntity.setEvents(real.getEvents()); - break; - default: - continue; + case CONFIGS: + finalEntity.setConfigs(real.getConfigs()); + break; + case METRICS: + finalEntity.setMetrics(real.getMetrics()); + break; + case INFO: + finalEntity.setInfo(real.getInfo()); + break; + case IS_RELATED_TO: + finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities()); + break; + case RELATES_TO: + finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities()); + break; + case EVENTS: + finalEntity.setEvents(real.getEvents()); + break; + default: + continue; } } } - private String getFlowRunPath(String userId, String clusterId, String flowName, - Long flowRunId, String appId) - throws IOException { + private String getFlowRunPath(String userId, String clusterId, + String flowName, Long flowRunId, String appId) throws IOException { if (userId != null && flowName != null && flowRunId != null) { return userId + "/" + flowName + "/" + flowRunId; } @@ -272,11 +272,11 @@ public class FileSystemTimelineReaderImpl extends AbstractService Map<Long, Set<TimelineEntity>> sortedEntities = new TreeMap<>( new Comparator<Long>() { - @Override - public int compare(Long l1, Long l2) { - return l2.compareTo(l1); - } + @Override + public int compare(Long l1, Long l2) { + return l2.compareTo(l1); } + } ); for (File entityFile : dir.listFiles()) { if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java index 4385bbc..74a03ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java @@ -76,9 +76,10 @@ public class FileSystemTimelineWriterImpl extends AbstractService return response; } - private synchronized void write(String clusterId, String userId, String flowName, - String flowVersion, long flowRun, String appId, TimelineEntity entity, - TimelineWriteResponse response) throws IOException { + private synchronized void write(String clusterId, String userId, + String flowName, String flowVersion, long flowRun, String appId, + TimelineEntity entity, TimelineWriteResponse response) + throws IOException { PrintWriter out = null; try { String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId, http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index 9f8257f..a384a84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -35,6 +35,9 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContex import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader; import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory; +/** + * HBase based implementation for {@link TimelineReader}. + */ public class HBaseTimelineReaderImpl extends AbstractService implements TimelineReader { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index 2a82ccb..997b175 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java @@ -93,7 +93,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } /** - * initializes the hbase connection to write to the entity table + * initializes the hbase connection to write to the entity table. */ @Override protected void serviceInit(Configuration conf) throws Exception { @@ -104,7 +104,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn); applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn); flowRunTable = new FlowRunTable().getTableMutator(hbaseConf, conn); - flowActivityTable = new FlowActivityTable().getTableMutator(hbaseConf, conn); + flowActivityTable = + new FlowActivityTable().getTableMutator(hbaseConf, conn); } /** @@ -289,7 +290,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } /** - * Stores the Relations from the {@linkplain TimelineEntity} object + * Stores the Relations from the {@linkplain TimelineEntity} object. */ private <T> void storeRelations(byte[] rowKey, Map<String, Set<String>> connectedEntities, @@ -306,7 +307,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } /** - * Stores information from the {@linkplain TimelineEntity} object + * Stores information from the {@linkplain TimelineEntity} object. */ private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion, boolean isApplication) throws IOException { @@ -341,7 +342,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } /** - * stores the config information from {@linkplain TimelineEntity} + * stores the config information from {@linkplain TimelineEntity}. */ private void storeConfig(byte[] rowKey, Map<String, String> config, boolean isApplication) throws IOException { @@ -351,17 +352,17 @@ public class HBaseTimelineWriterImpl extends AbstractService implements for (Map.Entry<String, String> entry : config.entrySet()) { if (isApplication) { ApplicationColumnPrefix.CONFIG.store(rowKey, applicationTable, - entry.getKey(), null, entry.getValue()); + entry.getKey(), null, entry.getValue()); } else { EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(), - null, entry.getValue()); + null, entry.getValue()); } } } /** * stores the {@linkplain TimelineMetric} information from the - * {@linkplain TimelineEvent} object + * {@linkplain TimelineEvent} object. */ private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics, boolean isApplication) throws IOException { @@ -373,10 +374,10 @@ public class HBaseTimelineWriterImpl extends AbstractService implements Long timestamp = timeseriesEntry.getKey(); if (isApplication) { ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable, - metricColumnQualifier, timestamp, timeseriesEntry.getValue()); + metricColumnQualifier, timestamp, timeseriesEntry.getValue()); } else { EntityColumnPrefix.METRIC.store(rowKey, entityTable, - metricColumnQualifier, timestamp, timeseriesEntry.getValue()); + metricColumnQualifier, timestamp, timeseriesEntry.getValue()); } } } @@ -384,7 +385,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } /** - * Stores the events from the {@linkplain TimelineEvent} object + * Stores the events from the {@linkplain TimelineEvent} object. */ private void storeEvents(byte[] rowKey, Set<TimelineEvent> events, boolean isApplication) throws IOException { @@ -428,10 +429,10 @@ public class HBaseTimelineWriterImpl extends AbstractService implements Bytes.toBytes(info.getKey())); if (isApplication) { ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, - compoundColumnQualifierBytes, null, info.getValue()); + compoundColumnQualifierBytes, null, info.getValue()); } else { EntityColumnPrefix.EVENT.store(rowKey, entityTable, - compoundColumnQualifierBytes, null, info.getValue()); + compoundColumnQualifierBytes, null, info.getValue()); } } // for info: eventInfo } @@ -459,7 +460,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements /** * close the hbase connections The close APIs perform flushing and release any - * resources held + * resources held. */ @Override protected void serviceStop() throws Exception { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java index e1219e0..1484f22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggre import java.io.IOException; /** - * YARN timeline service v2 offline aggregation storage interface + * YARN timeline service v2 offline aggregation storage interface. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -46,8 +46,8 @@ public abstract class OfflineAggregationWriter extends AbstractService { /** * Persist aggregated timeline entities to the offline store based on which - * track this entity is to be rolled up to. The tracks along which aggregations - * are to be done are given by {@link OfflineAggregationInfo}. + * track this entity is to be rolled up to. The tracks along which + * aggregations are to be done are given by {@link OfflineAggregationInfo}. * * @param context a {@link TimelineCollectorContext} object that describes the * context information of the aggregated data. Depends on the @@ -58,9 +58,10 @@ public abstract class OfflineAggregationWriter extends AbstractService { * detail of the aggregation. Current supported option is * {@link OfflineAggregationInfo#FLOW_AGGREGATION}. * @return a {@link TimelineWriteResponse} object. - * @throws IOException + * @throws IOException if any problem occurs while writing aggregated + * entities. */ abstract TimelineWriteResponse writeAggregatedEntity( - TimelineCollectorContext context, - TimelineEntities entities, OfflineAggregationInfo info) throws IOException; + TimelineCollectorContext context, TimelineEntities entities, + OfflineAggregationInfo info) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java index b5834c0..130cb6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java @@ -102,18 +102,18 @@ public class PhoenixOfflineAggregationWriterImpl private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER = "timeline_cf_placeholder"; - /** Default Phoenix JDBC driver name */ + /** Default Phoenix JDBC driver name. */ private static final String DRIVER_CLASS_NAME = "org.apache.phoenix.jdbc.PhoenixDriver"; - /** Default Phoenix timeline config column family */ + /** Default Phoenix timeline config column family. */ private static final String METRIC_COLUMN_FAMILY = "m."; - /** Default Phoenix timeline info column family */ + /** Default Phoenix timeline info column family. */ private static final String INFO_COLUMN_FAMILY = "i."; - /** Default separator for Phoenix storage */ + /** Default separator for Phoenix storage. */ private static final String AGGREGATION_STORAGE_SEPARATOR = ";"; - /** Connection string to the deployed Phoenix cluster */ + /** Connection string to the deployed Phoenix cluster. */ private String connString = null; private Properties connProperties = new Properties(); @@ -162,7 +162,8 @@ public class PhoenixOfflineAggregationWriterImpl } int idx = info.setStringsForPrimaryKey(ps, context, null, 1); ps.setLong(idx++, entity.getCreatedTime()); - ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(), + ps.setString(idx++, + StringUtils.join(formattedMetrics.keySet().toArray(), AGGREGATION_STORAGE_SEPARATOR)); ps.execute(); @@ -185,7 +186,7 @@ public class PhoenixOfflineAggregationWriterImpl * Create Phoenix tables for offline aggregation storage if the tables do not * exist. * - * @throws IOException + * @throws IOException if any problem happens while creating Phoenix tables. */ public void createPhoenixTables() throws IOException { // Create tables if necessary @@ -197,7 +198,8 @@ public class PhoenixOfflineAggregationWriterImpl + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, " + "flow_name VARCHAR NOT NULL, " + "created_time UNSIGNED_LONG, " - + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, " + + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + + " VARBINARY, " + "metric_names VARCHAR, info_keys VARCHAR " + "CONSTRAINT pk PRIMARY KEY(" + "user, cluster, flow_name))"; @@ -206,7 +208,8 @@ public class PhoenixOfflineAggregationWriterImpl + OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, " + "created_time UNSIGNED_LONG, " - + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, " + + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + + " VARBINARY, " + "metric_names VARCHAR, info_keys VARCHAR " + "CONSTRAINT pk PRIMARY KEY(user, cluster))"; stmt.executeUpdate(sql); @@ -251,9 +254,9 @@ public class PhoenixOfflineAggregationWriterImpl private static class DynamicColumns<K> { static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY"; static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR"; - String columnFamilyPrefix; - String type; - Set<K> columns; + private String columnFamilyPrefix; + private String type; + private Set<K> columns; public DynamicColumns(String columnFamilyPrefix, String type, Set<K> keyValues) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java index 955ca80..f0b1e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage; /** * specifies the tracks along which an entity - * info is to be aggregated on + * info is to be aggregated on. * */ public enum TimelineAggregationTrack { http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java index ccb33b7..e8eabf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java @@ -97,7 +97,8 @@ public interface TimelineReader extends Service { * <cite>FlowRunEntity</cite>.<br> * For all other entity types, entity returned is of type * <cite>TimelineEntity</cite>. - * @throws IOException + * @throws IOException if there is an exception encountered while fetching + * entity from backend storage. */ TimelineEntity getEntity(TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) throws IOException; @@ -169,7 +170,8 @@ public interface TimelineReader extends Service { * <cite>FlowRunEntity</cite>.<br> * For all other entity types, entities returned are of type * <cite>TimelineEntity</cite>. - * @throws IOException + * @throws IOException if there is an exception encountered while fetching + * entity from backend storage. */ Set<TimelineEntity> getEntities( TimelineReaderContext context, http://git-wip-us.apache.org/repos/asf/hadoop/blob/db76a3ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java index 46bc2e6..33f5449 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java @@ -53,7 +53,9 @@ import com.google.common.annotations.VisibleForTesting; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class TimelineSchemaCreator { +public final class TimelineSchemaCreator { + private TimelineSchemaCreator() { + } final static String NAME = TimelineSchemaCreator.class.getSimpleName(); private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
