Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 452BA200B4B for ; Thu, 21 Jul 2016 20:29:43 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 43B47160A7C; Thu, 21 Jul 2016 18:29:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 65D9E160A72 for ; Thu, 21 Jul 2016 20:29:42 +0200 (CEST) Received: (qmail 36424 invoked by uid 500); 21 Jul 2016 18:29:36 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 36415 invoked by uid 99); 21 Jul 2016 18:29:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Jul 2016 18:29:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1DEACE2C1A; Thu, 21 Jul 2016 18:29:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhz@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp. Date: Thu, 21 Jul 2016 18:29:36 +0000 (UTC) archived-at: Thu, 21 Jul 2016 18:29:43 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 2f14d585c -> 6a7fe835f HDFS-10653. Optimize conversion from path string to components. Contributed by Daryn Sharp. (cherry picked from commit bd3dcf46e263b6e6aa3fca6a5d9936cc49e3280f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a7fe835 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a7fe835 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a7fe835 Branch: refs/heads/branch-2.8 Commit: 6a7fe835f19141cc633824cd5d21f1e30f014bce Parents: 2f14d58 Author: Jing Zhao Authored: Thu Jul 21 11:14:39 2016 -0700 Committer: Zhe Zhang Committed: Thu Jul 21 11:29:27 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 9 ++++++ .../hadoop/hdfs/server/namenode/INode.java | 20 +++++-------- .../server/namenode/TestSnapshotPathINodes.java | 30 +++++++------------- 3 files changed, 26 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a7fe835/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 0848521..da7d26a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -321,6 +321,15 @@ public class DFSUtil { } /** + * Convert a UTF8 string to an array of byte arrays. + */ + public static byte[][] getPathComponents(String path) { + // avoid intermediate split to String[] + final byte[] bytes = string2Bytes(path); + return bytes2byteArray(bytes, bytes.length, (byte)Path.SEPARATOR_CHAR); + } + + /** * Splits the array of bytes into array of arrays of bytes * on byte separator * @param bytes the array of bytes to split http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a7fe835/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 9d04fbb..b548763 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -727,18 +727,8 @@ public abstract class INode implements INodeAttributes, Diff.Element { */ @VisibleForTesting public static byte[][] getPathComponents(String path) { - return getPathComponents(getPathNames(path)); - } - - /** Convert strings to byte arrays for path components. */ - static byte[][] getPathComponents(String[] strings) { - if (strings.length == 0) { - return new byte[][]{null}; - } - byte[][] bytes = new byte[strings.length][]; - for (int i = 0; i < strings.length; i++) - bytes[i] = DFSUtil.string2Bytes(strings[i]); - return bytes; + checkAbsolutePath(path); + return DFSUtil.getPathComponents(path); } /** @@ -747,11 +737,15 @@ public abstract class INode implements INodeAttributes, Diff.Element { * @return array of path components. */ public static String[] getPathNames(String path) { + checkAbsolutePath(path); + return StringUtils.split(path, Path.SEPARATOR_CHAR); + } + + private static void checkAbsolutePath(final String path) { if (path == null || !path.startsWith(Path.SEPARATOR)) { throw new AssertionError("Absolute path required, but got '" + path + "'"); } - return StringUtils.split(path, Path.SEPARATOR_CHAR); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a7fe835/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index e416e00..45c65ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -138,8 +138,7 @@ public class TestSnapshotPathINodes { @Test (timeout=15000) public void testNonSnapshotPathINodes() throws Exception { // Get the inodes by resolving the path of a normal file - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // The number of inodes should be equal to components.length @@ -175,8 +174,7 @@ public class TestSnapshotPathINodes { // The path when accessing the snapshot file of file1 is // /TestSnapshot/sub1/.snapshot/s1/file1 String snapshotPath = sub1.toString() + "/.snapshot/s1/file1"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore @@ -199,8 +197,7 @@ public class TestSnapshotPathINodes { // Resolve the path "/TestSnapshot/sub1/.snapshot" String dotSnapshotPath = sub1.toString() + "/.snapshot"; - names = INode.getPathNames(dotSnapshotPath); - components = INode.getPathComponents(names); + components = INode.getPathComponents(dotSnapshotPath); nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // The number of INodes returned should still be components.length // since we put a null in the inode array for ".snapshot" @@ -246,8 +243,7 @@ public class TestSnapshotPathINodes { // Resolve the path for the snapshot file // /TestSnapshot/sub1/.snapshot/s2/file1 String snapshotPath = sub1.toString() + "/.snapshot/s2/file1"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore @@ -264,8 +260,7 @@ public class TestSnapshotPathINodes { } // Check the INodes for path /TestSnapshot/sub1/file1 - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // The length of inodes should be equal to components.length @@ -314,8 +309,7 @@ public class TestSnapshotPathINodes { { // Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3 String snapshotPath = sub1.toString() + "/.snapshot/s4/file3"; - String[] names = INode.getPathNames(snapshotPath); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(snapshotPath); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // Length of inodes should be (components.length - 1), since we will ignore @@ -334,8 +328,7 @@ public class TestSnapshotPathINodes { } // Check the inodes for /TestSnapshot/sub1/file3 - String[] names = INode.getPathNames(file3.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file3.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // The number of inodes should be equal to components.length @@ -361,8 +354,7 @@ public class TestSnapshotPathINodes { @Test (timeout=15000) public void testSnapshotPathINodesAfterModification() throws Exception { // First check the INode for /TestSnapshot/sub1/file1 - String[] names = INode.getPathNames(file1.toString()); - byte[][] components = INode.getPathComponents(names); + byte[][] components = INode.getPathComponents(file1.toString()); INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // The number of inodes should be equal to components.length @@ -385,8 +377,7 @@ public class TestSnapshotPathINodes { // Check the INodes for snapshot of file1 String snapshotPath = sub1.toString() + "/.snapshot/s3/file1"; - names = INode.getPathNames(snapshotPath); - components = INode.getPathComponents(names); + components = INode.getPathComponents(snapshotPath); INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); // Length of ssInodes should be (components.length - 1), since we will @@ -404,8 +395,7 @@ public class TestSnapshotPathINodes { snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshotId())); // Check the INode for /TestSnapshot/sub1/file1 again - names = INode.getPathNames(file1.toString()); - components = INode.getPathComponents(names); + components = INode.getPathComponents(file1.toString()); INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); assertSnapshot(newNodesInPath, false, s3, -1); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org