hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject git commit: HADOOP-11035. distcp on mr1(branch-1) fails with NPE using a short relative source path. Contributed by Zhihai Xu.
Date Mon, 08 Sep 2014 19:44:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-1 38fd9e8d6 -> 4ca5ce88d


HADOOP-11035. distcp on mr1(branch-1) fails with NPE using a short relative source path. Contributed
by Zhihai Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ca5ce88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ca5ce88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ca5ce88

Branch: refs/heads/branch-1
Commit: 4ca5ce88d5e88fd7368fa94b4886231b1ae9bb26
Parents: 38fd9e8
Author: Andrew Wang <wang@apache.org>
Authored: Mon Sep 8 12:44:23 2014 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Mon Sep 8 12:44:23 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 .../org/apache/hadoop/fs/TestCopyFiles.java     | 40 +++++++++++++++++++-
 src/tools/org/apache/hadoop/tools/DistCp.java   |  6 ++-
 3 files changed, 45 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ca5ce88/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 18e2531..70678ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -245,6 +245,9 @@ Release 1.3.0 - unreleased
     MAPREDUCE-6063. Correct spill size calculation for spills wrapping the
     circular buffer. (zhihai xu via cdouglas)
 
+    HADOOP-11035. distcp on mr1(branch-1) fails with NPE using a short
+    relative source path. (zhihai xu via wang)
+
 Release 1.2.2 - unreleased
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ca5ce88/src/test/org/apache/hadoop/fs/TestCopyFiles.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/hadoop/fs/TestCopyFiles.java b/src/test/org/apache/hadoop/fs/TestCopyFiles.java
index 812fdfa..9ce0769 100644
--- a/src/test/org/apache/hadoop/fs/TestCopyFiles.java
+++ b/src/test/org/apache/hadoop/fs/TestCopyFiles.java
@@ -49,7 +49,6 @@ import org.apache.log4j.Level;
  * A JUnit test for copying files recursively.
  */
 public class TestCopyFiles extends TestCase {
-  
   private static final String JT_STAGING_AREA_ROOT = "mapreduce.jobtracker.staging.root.dir";
   private static final String JT_STAGING_AREA_ROOT_DEFAULT = "/tmp/hadoop/mapred/staging";
   private static final String FS_TRASH_INTERVAL_KEY = "fs.trash.interval";
@@ -271,7 +270,44 @@ public class TestCopyFiles extends TestCase {
     deldir(localfs, TEST_ROOT_DIR+"/destdat");
     deldir(localfs, TEST_ROOT_DIR+"/srcdat");
   }
-  
+
+  /** copy files from local file system to local file system
+   *  using relative short path name to verify
+   *  DistCp supports relative short path name */
+  public void testCopyFromLocalToLocalUsingRelativePathName() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem localfs = FileSystem.get(LOCAL_FS_URI, conf);
+    MyFile[] files = createFiles(LOCAL_FS_URI, TEST_ROOT_DIR+"/srcdat");
+
+    // get DistCp source and destination relative path
+    Path src = new Path(TEST_ROOT_DIR+"/srcdat");
+    Path srcRoot = new Path(".");
+    FileStatus srcfilestat = localfs.getFileStatus(src);
+    FileStatus srcRootfilestat = localfs.getFileStatus(srcRoot);
+    // srcStr is file:/{hadoop-src-root}/build/test/data/srcdat
+    String srcStr = srcfilestat.getPath().toString();
+    // srcRootStr is file:/{hadoop-src-root}
+    String srcRootStr = srcRootfilestat.getPath().toString();
+    // +1 to remove /, srcRelativePath is build/test/data/srcdat
+    String srcRelativePath = srcStr.substring(srcRootStr.length() + 1);
+    // replace "srcdat" with "destdat" to get destRelativePath,
+    // destRelativePath is build/test/data/destdat
+    String destRelativePath = srcRelativePath.substring(0,
+        srcRelativePath.length() - "srcdat".length()) + "destdat";
+
+    // run DistCp with relative path source and destination parameters
+    ToolRunner.run(new DistCp(new Configuration()),
+        new String[] {srcRelativePath, destRelativePath});
+
+    // check result
+    assertTrue("Source and destination directories do not match.",
+               checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
+
+    // clean up
+    deldir(localfs, TEST_ROOT_DIR+"/destdat");
+    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
+  }
+
   /** copy files from dfs file system to dfs file system */
   public void testCopyFromDfsToDfs() throws Exception {
     String namenode = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ca5ce88/src/tools/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/src/tools/org/apache/hadoop/tools/DistCp.java b/src/tools/org/apache/hadoop/tools/DistCp.java
index 1fe9f27..09122cf 100644
--- a/src/tools/org/apache/hadoop/tools/DistCp.java
+++ b/src/tools/org/apache/hadoop/tools/DistCp.java
@@ -1096,9 +1096,11 @@ public class DistCp implements Tool {
     final FileStatus jobDirStat = jobfs.getFileStatus(jobDirectory);
     try {
       for(Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
-        final Path src = srcItr.next();
+        Path src = srcItr.next();
         FileSystem srcfs = src.getFileSystem(conf);
         FileStatus srcfilestat = srcfs.getFileStatus(src);
+        // change src to full path to match child.getPath().
+        src = srcfilestat.getPath();
         Path root = special && srcfilestat.isDir()? src: src.getParent();
         final boolean needToFilterJobDir = srcfs.equals(jobfs);
 
@@ -1116,7 +1118,7 @@ public class DistCp implements Tool {
           FileStatus[] children = srcfs.listStatus(cur.getPath());
           for(int i = 0; i < children.length; i++) {
             boolean skipfile = false;
-            final FileStatus child = children[i]; 
+            final FileStatus child = children[i];
             final String dst = makeRelative(root, child.getPath());
 
             if (child.isDir()) {


Mime
View raw message