hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject hadoop git commit: HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp.
Date Fri, 22 Jul 2016 17:05:48 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 2f546755c -> b9d4144c2


HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp.

(cherry picked from commit bd3dcf46e263b6e6aa3fca6a5d9936cc49e3280f)
(cherry picked from commit 6a7fe835f19141cc633824cd5d21f1e30f014bce)
(cherry picked from commit 2a5b8e5b3a7ac716253f68ccd25f95ae189bc3ea)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9d4144c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9d4144c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9d4144c

Branch: refs/heads/branch-2.6
Commit: b9d4144c2b40bf29663c21633fb4008a1c1aeb8b
Parents: 2f54675
Author: Jing Zhao <jing9@apache.org>
Authored: Thu Jul 21 11:14:39 2016 -0700
Committer: Zhe Zhang <zhz@apache.org>
Committed: Fri Jul 22 08:47:17 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |  9 ++++++
 .../hdfs/server/namenode/FSNamesystem.java      |  9 +++---
 .../hadoop/hdfs/server/namenode/INode.java      | 20 +++++--------
 .../server/namenode/TestSnapshotPathINodes.java | 30 +++++++-------------
 4 files changed, 31 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 031896a..67880cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -417,6 +417,15 @@ public class DFSUtil {
   }
 
   /**
+   * Convert a UTF8 string to an array of byte arrays.
+   */
+  public static byte[][] getPathComponents(String path) {
+    // avoid intermediate split to String[]
+    final byte[] bytes = string2Bytes(path);
+    return bytes2byteArray(bytes, bytes.length, (byte)Path.SEPARATOR_CHAR);
+  }
+
+  /**
    * Splits the array of bytes into array of arrays of bytes
    * on byte separator
    * @param bytes the array of bytes to split

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index bd4f555..4a70ac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4327,8 +4327,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
                  UnresolvedLinkException, SnapshotAccessControlException,
                  AclException {
     src = FSDirectory.normalizePath(src);
-    String[] names = INode.getPathNames(src);
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(src);
     final int lastInodeIndex = components.length - 1;
 
     dir.writeLock();
@@ -4344,7 +4343,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       StringBuilder pathbuilder = new StringBuilder();
       int i = 1;
       for(; i < inodes.length && inodes[i] != null; i++) {
-        pathbuilder.append(Path.SEPARATOR).append(names[i]);
+        pathbuilder.append(Path.SEPARATOR).
+            append(DFSUtil.bytes2String(components[i]));
         if (!inodes[i].isDirectory()) {
           throw new FileAlreadyExistsException(
                   "Parent path is not a directory: "
@@ -4386,7 +4386,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
 
       // create directories beginning from the first null index
       for(; i < inodes.length; i++) {
-        pathbuilder.append(Path.SEPARATOR).append(names[i]);
+        pathbuilder.append(Path.SEPARATOR).
+            append(DFSUtil.bytes2String(components[i]));
         dir.unprotectedMkdir(allocateNewInodeId(), iip, i, components[i],
                 (i < lastInodeIndex) ? parentPermissions : permissions, null,
                 now);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 4454930..2349740 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -707,18 +707,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]>
{
    */
   @VisibleForTesting
   public static byte[][] getPathComponents(String path) {
-    return getPathComponents(getPathNames(path));
-  }
-
-  /** Convert strings to byte arrays for path components. */
-  static byte[][] getPathComponents(String[] strings) {
-    if (strings.length == 0) {
-      return new byte[][]{null};
-    }
-    byte[][] bytes = new byte[strings.length][];
-    for (int i = 0; i < strings.length; i++)
-      bytes[i] = DFSUtil.string2Bytes(strings[i]);
-    return bytes;
+    checkAbsolutePath(path);
+    return DFSUtil.getPathComponents(path);
   }
 
   /**
@@ -727,10 +717,14 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]>
{
    * @return array of path components.
    */
   public static String[] getPathNames(String path) {
+    checkAbsolutePath(path);
+    return StringUtils.split(path, Path.SEPARATOR_CHAR);
+  }
+
+  private static void checkAbsolutePath(final String path) {
     if (path == null || !path.startsWith(Path.SEPARATOR)) {
       throw new AssertionError("Absolute path required");
     }
-    return StringUtils.split(path, Path.SEPARATOR_CHAR);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9d4144c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
index 95524e8..1e9be70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
@@ -142,8 +142,7 @@ public class TestSnapshotPathINodes {
   @Test (timeout=15000)
   public void testNonSnapshotPathINodes() throws Exception {
     // Get the inodes by resolving the path of a normal file
-    String[] names = INode.getPathNames(file1.toString());
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(file1.toString());
     INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] inodes = nodesInPath.getINodes();
     // The number of inodes should be equal to components.length
@@ -192,8 +191,7 @@ public class TestSnapshotPathINodes {
     // The path when accessing the snapshot file of file1 is
     // /TestSnapshot/sub1/.snapshot/s1/file1
     String snapshotPath = sub1.toString() + "/.snapshot/s1/file1";
-    String[] names = INode.getPathNames(snapshotPath);
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(snapshotPath);
     INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] inodes = nodesInPath.getINodes();
     // Length of inodes should be (components.length - 1), since we will ignore
@@ -228,8 +226,7 @@ public class TestSnapshotPathINodes {
     
     // Resolve the path "/TestSnapshot/sub1/.snapshot"  
     String dotSnapshotPath = sub1.toString() + "/.snapshot";
-    names = INode.getPathNames(dotSnapshotPath);
-    components = INode.getPathComponents(names);
+    components = INode.getPathComponents(dotSnapshotPath);
     nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     inodes = nodesInPath.getINodes();
     // The number of INodes returned should be components.length - 1 since we
@@ -277,8 +274,7 @@ public class TestSnapshotPathINodes {
       // Resolve the path for the snapshot file
       // /TestSnapshot/sub1/.snapshot/s2/file1
       String snapshotPath = sub1.toString() + "/.snapshot/s2/file1";
-      String[] names = INode.getPathNames(snapshotPath);
-      byte[][] components = INode.getPathComponents(names);
+      byte[][] components = INode.getPathComponents(snapshotPath);
       INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
       INode[] inodes = nodesInPath.getINodes();
       // Length of inodes should be (components.length - 1), since we will ignore
@@ -295,8 +291,7 @@ public class TestSnapshotPathINodes {
     }
 
     // Check the INodes for path /TestSnapshot/sub1/file1
-    String[] names = INode.getPathNames(file1.toString());
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(file1.toString());
     INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] inodes = nodesInPath.getINodes();
     // The length of inodes should be equal to components.length
@@ -337,8 +332,7 @@ public class TestSnapshotPathINodes {
     {
       // Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3
       String snapshotPath = sub1.toString() + "/.snapshot/s4/file3";
-      String[] names = INode.getPathNames(snapshotPath);
-      byte[][] components = INode.getPathComponents(names);
+      byte[][] components = INode.getPathComponents(snapshotPath);
       INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
       INode[] inodes = nodesInPath.getINodes();
       // Length of inodes should be (components.length - 1), since we will ignore
@@ -357,8 +351,7 @@ public class TestSnapshotPathINodes {
     }
 
     // Check the inodes for /TestSnapshot/sub1/file3
-    String[] names = INode.getPathNames(file3.toString());
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(file3.toString());
     INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] inodes = nodesInPath.getINodes();
     // The number of inodes should be equal to components.length
@@ -385,8 +378,7 @@ public class TestSnapshotPathINodes {
   @Test (timeout=15000)
   public void testSnapshotPathINodesAfterModification() throws Exception {
     // First check the INode for /TestSnapshot/sub1/file1
-    String[] names = INode.getPathNames(file1.toString());
-    byte[][] components = INode.getPathComponents(names);
+    byte[][] components = INode.getPathComponents(file1.toString());
     INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] inodes = nodesInPath.getINodes();
     // The number of inodes should be equal to components.length
@@ -408,8 +400,7 @@ public class TestSnapshotPathINodes {
 
     // Check the INodes for snapshot of file1
     String snapshotPath = sub1.toString() + "/.snapshot/s3/file1";
-    names = INode.getPathNames(snapshotPath);
-    components = INode.getPathComponents(names);
+    components = INode.getPathComponents(snapshotPath);
     INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     INode[] ssInodes = ssNodesInPath.getINodes();
     // Length of ssInodes should be (components.length - 1), since we will
@@ -427,8 +418,7 @@ public class TestSnapshotPathINodes {
         snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshotId()));
 
     // Check the INode for /TestSnapshot/sub1/file1 again
-    names = INode.getPathNames(file1.toString());
-    components = INode.getPathComponents(names);
+    components = INode.getPathComponents(file1.toString());
     INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
     assertSnapshot(newNodesInPath, false, s3, -1);
     INode[] newInodes = newNodesInPath.getINodes();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message