hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oz...@apache.org
Subject hadoop git commit: YARN-3205. FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem with an old configuration. Contributed by Zhihai Xu.
Date Wed, 18 Mar 2015 02:56:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 85473cd61 -> b00b216a9


YARN-3205. FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem
with an old configuration. Contributed by Zhihai Xu.

(cherry picked from commit 3bc72cc16d8c7b8addd8f565523001dfcc32b891)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b00b216a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b00b216a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b00b216a

Branch: refs/heads/branch-2
Commit: b00b216a919417b2abd364d6433ad08fad7db134
Parents: 85473cd
Author: Tsuyoshi Ozawa <ozawa@apache.org>
Authored: Wed Mar 18 11:53:14 2015 +0900
Committer: Tsuyoshi Ozawa <ozawa@apache.org>
Committed: Wed Mar 18 11:53:40 2015 +0900

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../recovery/FileSystemRMStateStore.java        | 22 +++++++++++++++-----
 .../recovery/TestFSRMStateStore.java            |  5 +++++
 3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 69b3c43..4b81d04 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -24,6 +24,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks
     via jianhe)
 
+    YARN-3205 FileSystemRMStateStore should disable FileSystem Cache to avoid
+    get a Filesystem with an old configuration. (Zhihai Xu via ozawa)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 8147597..7652a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -84,7 +84,10 @@ public class FileSystemRMStateStore extends RMStateStore {
   protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
       "AMRMTokenSecretManagerNode";
 
+  @VisibleForTesting
   protected FileSystem fs;
+  @VisibleForTesting
+  protected Configuration fsConf;
 
   private Path rootDirPath;
   @Private
@@ -121,14 +124,23 @@ public class FileSystemRMStateStore extends RMStateStore {
     // create filesystem only now, as part of service-start. By this time, RM is
     // authenticated with kerberos so we are good to create a file-system
     // handle.
-    Configuration conf = new Configuration(getConfig());
-    conf.setBoolean("dfs.client.retry.policy.enabled", true);
+    fsConf = new Configuration(getConfig());
+    fsConf.setBoolean("dfs.client.retry.policy.enabled", true);
     String retryPolicy =
-        conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
+        fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
           YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC);
-    conf.set("dfs.client.retry.policy.spec", retryPolicy);
+    fsConf.set("dfs.client.retry.policy.spec", retryPolicy);
+
+    String scheme = fsWorkingPath.toUri().getScheme();
+    if (scheme == null) {
+      scheme = FileSystem.getDefaultUri(fsConf).getScheme();
+    }
+    if (scheme != null) {
+      String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
+      fsConf.setBoolean(disableCacheName, true);
+    }
 
-    fs = fsWorkingPath.getFileSystem(conf);
+    fs = fsWorkingPath.getFileSystem(fsConf);
     mkdirsWithRetries(rmDTSecretManagerRoot);
     mkdirsWithRetries(rmAppRoot);
     mkdirsWithRetries(amrmTokenSecretManagerRoot);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00b216a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index d2eddd6..c5f5578 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -106,6 +106,11 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
       this.store = new TestFileSystemRMStore(conf);
       Assert.assertEquals(store.getNumRetries(), 8);
       Assert.assertEquals(store.getRetryInterval(), 900L);
+      Assert.assertTrue(store.fs.getConf() == store.fsConf);
+      FileSystem previousFs = store.fs;
+      store.startInternal();
+      Assert.assertTrue(store.fs != previousFs);
+      Assert.assertTrue(store.fs.getConf() == store.fsConf);
       return store;
     }
 


Mime
View raw message