hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [01/11] git commit: YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed by Zhijie Shen
Date Sat, 20 Sep 2014 17:35:34 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-6581 222bf0fe6 -> 09dab88d3


YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed
by Zhijie Shen


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/444acf8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/444acf8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/444acf8e

Branch: refs/heads/HDFS-6581
Commit: 444acf8ea795e4bc782f1ce3b5ef7a1a47d1d27d
Parents: 25fd69a
Author: Jian He <jianhe@apache.org>
Authored: Fri Sep 19 11:26:29 2014 -0700
Committer: Jian He <jianhe@apache.org>
Committed: Fri Sep 19 11:26:29 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../ApplicationHistoryServer.java               |  4 +-
 .../ahs/RMApplicationHistoryWriter.java         | 49 +++++++++++---------
 .../ahs/TestRMApplicationHistoryWriter.java     | 18 +++++++
 4 files changed, 50 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8f717ae..7175b14 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -400,6 +400,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2568. Fixed the potential test failures due to race conditions when RM
     work-preserving recovery is enabled. (Jian He via zjshen)
 
+    YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless
+    explicitly set. (Zhijie Shen via jianhe)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 204e118..f52ab07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -182,7 +182,9 @@ public class ApplicationHistoryServer extends CompositeService {
     // APPLICATION_HISTORY_STORE is neither null nor empty, it means that the
     // user has enabled it explicitly.
     if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
-        conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) {
+        conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
+        conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
+            NullApplicationHistoryStore.class.getName())) {
       return new ApplicationHistoryManagerOnTimelineStore(
           timelineDataManager, aclsManager);
     } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
index 58d2e3d..bd328ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
@@ -52,6 +52,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * <p>
  * {@link ResourceManager} uses this class to write the information of
@@ -71,8 +73,10 @@ public class RMApplicationHistoryWriter extends CompositeService {
     .getLog(RMApplicationHistoryWriter.class);
 
   private Dispatcher dispatcher;
-  private ApplicationHistoryWriter writer;
-  private boolean historyServiceEnabled;
+  @VisibleForTesting
+  ApplicationHistoryWriter writer;
+  @VisibleForTesting
+  boolean historyServiceEnabled;
 
   public RMApplicationHistoryWriter() {
     super(RMApplicationHistoryWriter.class.getName());
@@ -80,13 +84,18 @@ public class RMApplicationHistoryWriter extends CompositeService {
 
   @Override
   protected synchronized void serviceInit(Configuration conf) throws Exception {
-
     historyServiceEnabled =
         conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
           YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
+    if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
+        conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
+        conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
+            NullApplicationHistoryStore.class.getName())) {
+      historyServiceEnabled = false;
+    }
 
-    // Only create the services when the history service is enabled, preventing
-    // wasting the system resources.
+    // Only create the services when the history service is enabled and not
+    // using the null store, preventing wasting the system resources.
     if (historyServiceEnabled) {
       writer = createApplicationHistoryStore(conf);
       addIfService(writer);
@@ -112,25 +121,19 @@ public class RMApplicationHistoryWriter extends CompositeService {
 
   protected ApplicationHistoryStore createApplicationHistoryStore(
       Configuration conf) {
-    // If the history writer is not enabled, a dummy store will be used to
-    // write nothing
-    if (historyServiceEnabled) {
-      try {
-        Class<? extends ApplicationHistoryStore> storeClass =
-            conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
-              FileSystemApplicationHistoryStore.class,
+    try {
+      Class<? extends ApplicationHistoryStore> storeClass =
+          conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+              NullApplicationHistoryStore.class,
               ApplicationHistoryStore.class);
-        return storeClass.newInstance();
-      } catch (Exception e) {
-        String msg =
-            "Could not instantiate ApplicationHistoryWriter: "
-                + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
-                  FileSystemApplicationHistoryStore.class.getName());
-        LOG.error(msg, e);
-        throw new YarnRuntimeException(msg, e);
-      }
-    } else {
-      return new NullApplicationHistoryStore();
+      return storeClass.newInstance();
+    } catch (Exception e) {
+      String msg =
+          "Could not instantiate ApplicationHistoryWriter: "
+              + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
+                  NullApplicationHistoryStore.class.getName());
+      LOG.error(msg, e);
+      throw new YarnRuntimeException(msg, e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
index e83a6b9..1c90ad2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
@@ -78,6 +78,8 @@ public class TestRMApplicationHistoryWriter {
     store = new MemoryApplicationHistoryStore();
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+    conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+        MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
     writer = new RMApplicationHistoryWriter() {
 
       @Override
@@ -175,6 +177,22 @@ public class TestRMApplicationHistoryWriter {
   }
 
   @Test
+  public void testDefaultStoreSetup() throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+    RMApplicationHistoryWriter writer = new RMApplicationHistoryWriter();
+    writer.init(conf);
+    writer.start();
+    try {
+      Assert.assertFalse(writer.historyServiceEnabled);
+      Assert.assertNull(writer.writer);
+    } finally {
+      writer.stop();
+      writer.close();
+    }
+  }
+
+  @Test
   public void testWriteApplication() throws Exception {
     RMApp app = createRMApp(ApplicationId.newInstance(0, 1));
 


Mime
View raw message