YARN-6446. Revisit ATSv2 integration to ensure all required information is published. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbc0b975 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbc0b975 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbc0b975 Branch: refs/heads/yarn-native-services Commit: fbc0b9755db644e57aa1ff5e3f7525e8102fc72e Parents: 81cb532 Author: Jian He <jia...@apache.org> Authored: Thu May 4 09:15:01 2017 -0700 Committer: Jian He <jia...@apache.org> Committed: Tue Jul 25 10:49:25 2017 -0700 ---------------------------------------------------------------------- .../apache/slider/providers/ProviderService.java | 8 ++++++++ .../providers/docker/DockerProviderService.java | 10 ++++++++++ .../slider/server/appmaster/SliderAppMaster.java | 4 +++- .../ServiceTimelinePublisher.java | 8 +++++++- .../SliderTimelineMetricsConstants.java | 2 ++ .../TestServiceTimelinePublisher.java | 19 +++++++++++++++---- 6 files changed, 45 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 7e92bfa..edd313b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -28,6 +28,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import java.io.IOException; @@ -57,4 +58,11 @@ public interface ProviderService extends Service { */ boolean processContainerStatus(ContainerId containerId, ContainerStatus status); + + /** + * Set service publisher. + * @param serviceTimelinePublisher service publisher. + */ + void setServiceTimelinePublisher( + ServiceTimelinePublisher serviceTimelinePublisher); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 93a481c..482bb27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -38,6 +38,7 @@ import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +58,7 @@ public class DockerProviderService extends AbstractService private static final String QUICK_LINKS = "quicklinks"; protected StateAccessForProviders amState; protected YarnRegistryViewForProviders yarnRegistry; + private ServiceTimelinePublisher serviceTimelinePublisher; protected DockerProviderService() { super("DockerProviderService"); @@ -126,6 +128,9 @@ public class DockerProviderService extends AbstractService PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, application.getQuicklinks().entrySet()); amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); + if (serviceTimelinePublisher != null) { + serviceTimelinePublisher.serviceAttemptUpdated(application); + } } public boolean processContainerStatus(ContainerId containerId, @@ -155,4 +160,9 @@ public class DockerProviderService extends AbstractService } return false; } + + @Override + public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { + this.serviceTimelinePublisher = publisher; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 1f379ea..ffa07fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -661,11 +661,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService asyncRMClient.registerTimelineV2Client(timelineClient); timelineClient.init(getConfig()); timelineClient.start(); - log.info("Timeline client started."); + log.info("Timeline v2 client started."); serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient); serviceTimelinePublisher.init(getConfig()); serviceTimelinePublisher.start(); + + providerService.setServiceTimelinePublisher(serviceTimelinePublisher); appState.setServiceTimelinePublisher(serviceTimelinePublisher); log.info("ServiceTimelinePublisher started."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java index 0e04af5..7504140 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java @@ -31,7 +31,6 @@ import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineV2Client; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.slider.api.resource.Application; @@ -109,6 +108,13 @@ public class ServiceTimelinePublisher extends CompositeService { publishComponents(application.getComponents()); } + public void serviceAttemptUpdated(Application application) { + TimelineEntity entity = createServiceAttemptEntity(application.getId()); + entity.addInfo(SliderTimelineMetricsConstants.QUICK_LINKS, + application.getQuicklinks()); + putEntity(entity); + } + public void serviceAttemptUnregistered(AppState appState, ActionStopSlider stopAction) { long currentTimeMillis = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java index 23e059d..58d77ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java @@ -37,6 +37,8 @@ public final class SliderTimelineMetricsConstants { public static final String LAUNCH_TIME = "LAUNCH_TIME"; + public static final String QUICK_LINKS = "QUICK_LINKS"; + public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND"; public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbc0b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java index 9f3ebba..e4fcbe8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java @@ -23,10 +23,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineV2Client; -import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.ApplicationState; @@ -77,6 +76,8 @@ public class TestServiceTimelinePublisher { @Before public void setUp() throws Exception { config = new Configuration(); + config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); timelineClient = new DummyTimelineClient(ApplicationId.fromString(SERVICEID)); serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient); @@ -88,8 +89,12 @@ public class TestServiceTimelinePublisher { @After public void tearDown() throws Exception { - serviceTimelinePublisher.stop(); - timelineClient.stop(); + if (serviceTimelinePublisher != null) { + serviceTimelinePublisher.stop(); + } + if (timelineClient != null) { + timelineClient.stop(); + } } @Test @@ -264,6 +269,12 @@ public class TestServiceTimelinePublisher { @Override public void putEntitiesAsync(TimelineEntity... entities) throws IOException, YarnException { + putEntities(entities); + } + + @Override + public void putEntities(TimelineEntity... entities) + throws IOException, YarnException { for (TimelineEntity timelineEntity : entities) { TimelineEntity entity = lastPublishedEntities.get(timelineEntity.getIdentifier()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
