hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [25/50] [abbrv] hadoop git commit: HDFS-13453. RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table. Contributed by Dibyendu Karmakar.
Date Wed, 02 May 2018 19:22:00 GMT
HDFS-13453. RBF: getMountPointDates should fetch latest subdir time/date when parent dir is
not present but /parent/child dirs are present in mount table. Contributed by Dibyendu Karmakar.

(cherry picked from commit 1134af9ad1daf683204df8f95a8f03d7baaa74d4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3c17359
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3c17359
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3c17359

Branch: refs/heads/YARN-8200
Commit: a3c17359513f775e5768ad9c57c33f3e2fdc6bdd
Parents: 2444d70
Author: Inigo Goiri <inigoiri@apache.org>
Authored: Thu Apr 19 14:56:36 2018 -0700
Committer: Inigo Goiri <inigoiri@apache.org>
Committed: Thu Apr 19 14:58:02 2018 -0700

----------------------------------------------------------------------
 .../federation/router/RouterRpcServer.java      | 55 ++++++++++++++++----
 .../federation/router/TestRouterMountTable.java | 21 +++++++-
 2 files changed, 65 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index d626699..8a0cf27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -2197,18 +2197,11 @@ public class RouterRpcServer extends AbstractService
   private Map<String, Long> getMountPointDates(String path) {
     Map<String, Long> ret = new TreeMap<>();
     if (subclusterResolver instanceof MountTableResolver) {
-      MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
-      String srcPath;
       try {
         final List<String> children = subclusterResolver.getMountPoints(path);
         for (String child : children) {
-          if (path.equals(Path.SEPARATOR)) {
-            srcPath = Path.SEPARATOR + child;
-          } else {
-            srcPath = path + Path.SEPARATOR + child;
-          }
-          MountTable entry = mountTable.getMountPoint(srcPath);
-          ret.put(child, entry.getDateModified());
+          Long modTime = getModifiedTime(ret, path, child);
+          ret.put(child, modTime);
         }
       } catch (IOException e) {
         LOG.error("Cannot get mount point", e);
@@ -2218,6 +2211,50 @@ public class RouterRpcServer extends AbstractService
   }
 
   /**
+   * Get modified time for child. If the child is present in mount table it
+   * will return the modified time. If the child is not present but subdirs of
+   * this child are present then it will return latest modified subdir's time
+   * as modified time of the requested child.
+   * @param ret contains children and modified times.
+   * @param mountTable.
+   * @param path Name of the path to start checking dates from.
+   * @param child child of the requested path.
+   * @return modified time.
+   */
+  private long getModifiedTime(Map<String, Long> ret, String path,
+      String child) {
+    MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+    String srcPath;
+    if (path.equals(Path.SEPARATOR)) {
+      srcPath = Path.SEPARATOR + child;
+    } else {
+      srcPath = path + Path.SEPARATOR + child;
+    }
+    Long modTime = 0L;
+    try {
+      // Get mount table entry for the srcPath
+      MountTable entry = mountTable.getMountPoint(srcPath);
+      // if srcPath is not in mount table but its subdirs are in mount
+      // table we will display latest modified subdir date/time.
+      if (entry == null) {
+        List<MountTable> entries = mountTable.getMounts(srcPath);
+        for (MountTable eachEntry : entries) {
+          // Get the latest date
+          if (ret.get(child) == null ||
+              ret.get(child) < eachEntry.getDateModified()) {
+            modTime = eachEntry.getDateModified();
+          }
+        }
+      } else {
+        modTime = entry.getDateModified();
+      }
+    } catch (IOException e) {
+      LOG.error("Cannot get mount point", e);
+    }
+    return modTime;
+  }
+
+  /**
    * Create a new file status for a mount point.
    *
    * @param name Name of the mount point.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3c17359/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
index b33b998..4d8ffe1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -167,6 +168,12 @@ public class TestRouterMountTable {
     addEntry = MountTable.newInstance(
         "/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
     assertTrue(addMountTable(addEntry));
+    addEntry = MountTable.newInstance(
+        "/testdir3/subdir1", Collections.singletonMap("ns0", "/testdir3"));
+    assertTrue(addMountTable(addEntry));
+    addEntry = MountTable.newInstance(
+        "/testA/testB/testC/testD", Collections.singletonMap("ns0", "/test"));
+    assertTrue(addMountTable(addEntry));
 
     // Create test dir in NN
     final FileSystem nnFs = nnContext.getFileSystem();
@@ -174,8 +181,18 @@ public class TestRouterMountTable {
 
     Map<String, Long> pathModTime = new TreeMap<>();
     for (String mount : mountTable.getMountPoints("/")) {
-      pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
-          .getDateModified());
+      if (mountTable.getMountPoint("/"+mount) != null) {
+        pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
+            .getDateModified());
+      } else {
+        List<MountTable> entries = mountTable.getMounts("/"+mount);
+        for (MountTable entry : entries) {
+          if (pathModTime.get(mount) == null ||
+              pathModTime.get(mount) < entry.getDateModified()) {
+            pathModTime.put(mount, entry.getDateModified());
+          }
+        }
+      }
     }
     FileStatus[] iterator = nnFs.listStatus(new Path("/"));
     for (FileStatus file : iterator) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message