hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [21/52] [abbrv] hadoop git commit: YARN-6446. Revisit ATSv2 integration to ensure all required information is published. Contributed by Rohith Sharma K S
Date Fri, 21 Jul 2017 18:39:23 GMT
YARN-6446. Revisit ATSv2 integration to ensure all required information is published. Contributed
by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8be2e50f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8be2e50f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8be2e50f

Branch: refs/heads/yarn-native-services
Commit: 8be2e50f6922931dca7b3a1e370dc5781d745125
Parents: 7f037ee
Author: Jian He <jianhe@apache.org>
Authored: Thu May 4 09:15:01 2017 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Fri Jul 21 11:38:13 2017 -0700

----------------------------------------------------------------------
 .../apache/slider/providers/ProviderService.java |  8 ++++++++
 .../providers/docker/DockerProviderService.java  | 10 ++++++++++
 .../slider/server/appmaster/SliderAppMaster.java |  4 +++-
 .../ServiceTimelinePublisher.java                |  8 +++++++-
 .../SliderTimelineMetricsConstants.java          |  2 ++
 .../TestServiceTimelinePublisher.java            | 19 +++++++++++++++----
 6 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 7e92bfa..edd313b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -28,6 +28,7 @@ import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
 import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
 
 import java.io.IOException;
@@ -57,4 +58,11 @@ public interface ProviderService extends Service {
    */
   boolean processContainerStatus(ContainerId containerId,
       ContainerStatus status);
+
+  /**
+   * Set service publisher.
+   * @param serviceTimelinePublisher service publisher.
+   */
+  void setServiceTimelinePublisher(
+      ServiceTimelinePublisher serviceTimelinePublisher);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 93a481c..482bb27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -38,6 +38,7 @@ import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.ProviderUtils;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
 import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,6 +58,7 @@ public class DockerProviderService extends AbstractService
   private static final String QUICK_LINKS = "quicklinks";
   protected StateAccessForProviders amState;
   protected YarnRegistryViewForProviders yarnRegistry;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
 
   protected DockerProviderService() {
     super("DockerProviderService");
@@ -126,6 +128,9 @@ public class DockerProviderService extends AbstractService
     PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
         application.getQuicklinks().entrySet());
     amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
+    if (serviceTimelinePublisher != null) {
+      serviceTimelinePublisher.serviceAttemptUpdated(application);
+    }
   }
 
   public boolean processContainerStatus(ContainerId containerId,
@@ -155,4 +160,9 @@ public class DockerProviderService extends AbstractService
     }
     return false;
   }
+
+  @Override
+  public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
+    this.serviceTimelinePublisher = publisher;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1f379ea..ffa07fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -661,11 +661,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
         asyncRMClient.registerTimelineV2Client(timelineClient);
         timelineClient.init(getConfig());
         timelineClient.start();
-        log.info("Timeline client started.");
+        log.info("Timeline v2 client started.");
 
         serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
         serviceTimelinePublisher.init(getConfig());
         serviceTimelinePublisher.start();
+
+        providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
         appState.setServiceTimelinePublisher(serviceTimelinePublisher);
         log.info("ServiceTimelinePublisher started.");
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
index 0e04af5..7504140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.slider.api.resource.Application;
@@ -109,6 +108,13 @@ public class ServiceTimelinePublisher extends CompositeService {
     publishComponents(application.getComponents());
   }
 
+  public void serviceAttemptUpdated(Application application) {
+    TimelineEntity entity = createServiceAttemptEntity(application.getId());
+    entity.addInfo(SliderTimelineMetricsConstants.QUICK_LINKS,
+        application.getQuicklinks());
+    putEntity(entity);
+  }
+
   public void serviceAttemptUnregistered(AppState appState,
       ActionStopSlider stopAction) {
     long currentTimeMillis = System.currentTimeMillis();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
index 23e059d..58d77ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/SliderTimelineMetricsConstants.java
@@ -37,6 +37,8 @@ public final class SliderTimelineMetricsConstants {
 
   public static final String LAUNCH_TIME = "LAUNCH_TIME";
 
+  public static final String QUICK_LINKS = "QUICK_LINKS";
+
   public static final String LAUNCH_COMMAND = "LAUNCH_COMMAND";
 
   public static final String TOTAL_CONTAINERS = "NUMBER_OF_CONTAINERS";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8be2e50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
index 9f3ebba..e4fcbe8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/timelineservice/TestServiceTimelinePublisher.java
@@ -23,10 +23,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
 import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.ApplicationState;
@@ -77,6 +76,8 @@ public class TestServiceTimelinePublisher {
   @Before
   public void setUp() throws Exception {
     config = new Configuration();
+    config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
     timelineClient =
         new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
     serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
@@ -88,8 +89,12 @@ public class TestServiceTimelinePublisher {
 
   @After
   public void tearDown() throws Exception {
-    serviceTimelinePublisher.stop();
-    timelineClient.stop();
+    if (serviceTimelinePublisher != null) {
+      serviceTimelinePublisher.stop();
+    }
+    if (timelineClient != null) {
+      timelineClient.stop();
+    }
   }
 
   @Test
@@ -264,6 +269,12 @@ public class TestServiceTimelinePublisher {
     @Override
     public void putEntitiesAsync(TimelineEntity... entities)
         throws IOException, YarnException {
+      putEntities(entities);
+    }
+
+    @Override
+    public void putEntities(TimelineEntity... entities)
+        throws IOException, YarnException {
       for (TimelineEntity timelineEntity : entities) {
         TimelineEntity entity =
             lastPublishedEntities.get(timelineEntity.getIdentifier());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message