hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject hadoop git commit: YARN-2375. Allow enabling/disabling timeline server per framework. (Mit Desai via jeagles)
Date Fri, 21 Nov 2014 05:35:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk c95b878ab -> c298a9a84


YARN-2375. Allow enabling/disabling timeline server per framework. (Mit Desai via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c298a9a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c298a9a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c298a9a8

Branch: refs/heads/trunk
Commit: c298a9a845f89317eb9efad332e6657c56736a4d
Parents: c95b878
Author: Jonathan Eagles <jeagles@gmail.com>
Authored: Thu Nov 20 23:34:35 2014 -0600
Committer: Jonathan Eagles <jeagles@gmail.com>
Committed: Thu Nov 20 23:34:35 2014 -0600

----------------------------------------------------------------------
 .../jobhistory/JobHistoryEventHandler.java      | 13 +++-
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../distributedshell/ApplicationMaster.java     | 47 ++++++++----
 .../client/api/impl/TimelineClientImpl.java     | 76 ++++++++------------
 .../client/api/impl/TestTimelineClient.java     | 35 ---------
 5 files changed, 74 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c298a9a8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 184baaa..45ddb9e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -243,9 +244,15 @@ public class JobHistoryEventHandler extends AbstractService
 
     if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
         MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
-      timelineClient = TimelineClient.createTimelineClient();
-      timelineClient.init(conf);
-      LOG.info("Emitting job history data to the timeline server is enabled");
+      if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+        timelineClient = TimelineClient.createTimelineClient();
+        timelineClient.init(conf);
+        LOG.info("Timeline service is enabled");
+        LOG.info("Emitting job history data to the timeline server is enabled");
+      } else {
+        LOG.info("Timeline service is not enabled");
+      }
     } else {
       LOG.info("Emitting job history data to the timeline server is not enabled");
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c298a9a8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index deb4072..6d86d75 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -81,6 +81,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2802. ClusterMetrics to include AM launch and register delays.
     (Zhihai Xu via kasha)
 
+    YARN-2375. Allow enabling/disabling timeline server per framework.
+    (Mit Desai via jeagles)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c298a9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 7906e6f..b3ea865 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -497,10 +497,16 @@ public class ApplicationMaster {
     requestPriority = Integer.parseInt(cliParser
         .getOptionValue("priority", "0"));
 
-    // Creating the Timeline Client
-    timelineClient = TimelineClient.createTimelineClient();
-    timelineClient.init(conf);
-    timelineClient.start();
+    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+      // Creating the Timeline Client
+      timelineClient = TimelineClient.createTimelineClient();
+      timelineClient.init(conf);
+      timelineClient.start();
+    } else {
+      timelineClient = null;
+      LOG.warn("Timeline service is not enabled");
+    }
 
     return true;
   }
@@ -548,9 +554,11 @@ public class ApplicationMaster {
     appSubmitterUgi =
         UserGroupInformation.createRemoteUser(appSubmitterUserName);
     appSubmitterUgi.addCredentials(credentials);
-
-    publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
-        DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+    
+    if(timelineClient != null) {
+      publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+          DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+    }
 
     AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
     amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
@@ -617,8 +625,10 @@ public class ApplicationMaster {
     }
     numRequestedContainers.set(numTotalContainers);
 
-    publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
-        DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
+    if(timelineClient != null) {
+      publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+          DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
+    }
   }
 
   @VisibleForTesting
@@ -681,6 +691,11 @@ public class ApplicationMaster {
     
     amRMClient.stop();
 
+    // Stop Timeline Client
+    if(timelineClient != null) {
+      timelineClient.stop();
+    }
+
     return success;
   }
   
@@ -724,8 +739,10 @@ public class ApplicationMaster {
           LOG.info("Container completed successfully." + ", containerId="
               + containerStatus.getContainerId());
         }
-        publishContainerEndEvent(
-            timelineClient, containerStatus, domainId, appSubmitterUgi);
+        if(timelineClient != null) {
+          publishContainerEndEvent(
+              timelineClient, containerStatus, domainId, appSubmitterUgi);
+        }
       }
       
       // ask for more containers if any failed
@@ -840,9 +857,11 @@ public class ApplicationMaster {
       if (container != null) {
         applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
       }
-      ApplicationMaster.publishContainerStartEvent(
-          applicationMaster.timelineClient, container,
-          applicationMaster.domainId, applicationMaster.appSubmitterUgi);
+      if(applicationMaster.timelineClient != null) {
+        ApplicationMaster.publishContainerStartEvent(
+            applicationMaster.timelineClient, container,
+            applicationMaster.domainId, applicationMaster.appSubmitterUgi);
+      }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c298a9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 2028cc9..78901c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -105,7 +105,6 @@ public class TimelineClientImpl extends TimelineClient {
   private DelegationTokenAuthenticator authenticator;
   private DelegationTokenAuthenticatedURL.Token token;
   private URI resURI;
-  private boolean isEnabled;
 
   @Private
   @VisibleForTesting
@@ -247,55 +246,42 @@ public class TimelineClientImpl extends TimelineClient {
   }
 
   protected void serviceInit(Configuration conf) throws Exception {
-    isEnabled = conf.getBoolean(
-        YarnConfiguration.TIMELINE_SERVICE_ENABLED,
-        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
-    if (!isEnabled) {
-      LOG.info("Timeline service is not enabled");
+    ClientConfig cc = new DefaultClientConfig();
+    cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+    connConfigurator = newConnConfigurator(conf);
+    if (UserGroupInformation.isSecurityEnabled()) {
+      authenticator = new KerberosDelegationTokenAuthenticator();
     } else {
-      ClientConfig cc = new DefaultClientConfig();
-      cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
-      connConfigurator = newConnConfigurator(conf);
-      if (UserGroupInformation.isSecurityEnabled()) {
-        authenticator = new KerberosDelegationTokenAuthenticator();
-      } else {
-        authenticator = new PseudoDelegationTokenAuthenticator();
-      }
-      authenticator.setConnectionConfigurator(connConfigurator);
-      token = new DelegationTokenAuthenticatedURL.Token();
-
-      connectionRetry = new TimelineClientConnectionRetry(conf);
-      client = new Client(new URLConnectionClientHandler(
-          new TimelineURLConnectionFactory()), cc);
-      TimelineJerseyRetryFilter retryFilter = new TimelineJerseyRetryFilter();
-      client.addFilter(retryFilter);
-
-      if (YarnConfiguration.useHttps(conf)) {
-        resURI = URI
-            .create(JOINER.join("https://", conf.get(
-                YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
-                YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
-                RESOURCE_URI_STR));
-      } else {
-        resURI = URI.create(JOINER.join("http://", conf.get(
-            YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
-            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS),
-            RESOURCE_URI_STR));
-      }
-      LOG.info("Timeline service address: " + resURI);
+      authenticator = new PseudoDelegationTokenAuthenticator();
+    }
+    authenticator.setConnectionConfigurator(connConfigurator);
+    token = new DelegationTokenAuthenticatedURL.Token();
+
+    connectionRetry = new TimelineClientConnectionRetry(conf);
+    client = new Client(new URLConnectionClientHandler(
+        new TimelineURLConnectionFactory()), cc);
+    TimelineJerseyRetryFilter retryFilter = new TimelineJerseyRetryFilter();
+    client.addFilter(retryFilter);
+
+    if (YarnConfiguration.useHttps(conf)) {
+      resURI = URI
+          .create(JOINER.join("https://", conf.get(
+              YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+              YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
+              RESOURCE_URI_STR));
+    } else {
+      resURI = URI.create(JOINER.join("http://", conf.get(
+          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS),
+          RESOURCE_URI_STR));
     }
+    LOG.info("Timeline service address: " + resURI);
     super.serviceInit(conf);
   }
 
   @Override
   public TimelinePutResponse putEntities(
       TimelineEntity... entities) throws IOException, YarnException {
-    if (!isEnabled) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Nothing will be put because timeline service is not enabled");
-      }
-      return new TimelinePutResponse();
-    }
     TimelineEntities entitiesContainer = new TimelineEntities();
     entitiesContainer.addEntities(Arrays.asList(entities));
     ClientResponse resp = doPosting(entitiesContainer, null);
@@ -306,12 +292,6 @@ public class TimelineClientImpl extends TimelineClient {
   @Override
   public void putDomain(TimelineDomain domain) throws IOException,
       YarnException {
-    if (!isEnabled) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Nothing will be put because timeline service is not enabled");
-      }
-      return;
-    }
     doPosting(domain, "domain");
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c298a9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index fe66e74..3b4d6bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -120,41 +120,6 @@ public class TestTimelineClient {
   }
 
   @Test
-  public void testPostEntitiesTimelineServiceNotEnabled() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
-    TimelineClientImpl client = createTimelineClient(conf);
-    mockEntityClientResponse(
-        client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
-    try {
-      TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(0, response.getErrors().size());
-    } catch (YarnException e) {
-      Assert.fail(
-          "putEntities should already return before throwing the exception");
-    }
-  }
-
-  @Test
-  public void testPostEntitiesTimelineServiceDefaultNotEnabled()
-      throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    // Unset the timeline service's enabled properties.
-    // Make sure default value is pickup up
-    conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED);
-    TimelineClientImpl client = createTimelineClient(conf);
-    mockEntityClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
-        false, false);
-    try {
-      TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(0, response.getErrors().size());
-    } catch (YarnException e) {
-      Assert
-          .fail("putEntities should already return before throwing the exception");
-    }
-  }
-
-  @Test
   public void testPutDomain() throws Exception {
     mockDomainClientResponse(client, ClientResponse.Status.OK, false);
     try {


Mime
View raw message