hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r638094 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/util/CopyFiles.java src/test/org/apache/hadoop/fs/TestCopyFiles.java
Date Mon, 17 Mar 2008 22:10:19 GMT
Author: cdouglas
Date: Mon Mar 17 15:10:16 2008
New Revision: 638094

URL: http://svn.apache.org/viewvc?rev=638094&view=rev
Log:
HADOOP-3011. Prohibit distcp from overwriting directories on the
destination filesystem with files.


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
    hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Mar 17 15:10:16 2008
@@ -263,6 +263,13 @@
     HADOOP-3006. Fix wrong packet size reported by DataNode when a block
     is being replicated. (rangadi)
     
+Release 0.16.2 - Unreleased
+
+  BUG FIXES
+
+    HADOOP-3011. Prohibit distcp from overwriting directories on the
+    destination filesystem with files. (cdouglas)
+
 Release 0.16.1 - 2008-03-13
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Mon Mar 17 15:10:16 2008
@@ -333,8 +333,16 @@
       else {
         if (totfiles == 1) {
           // Copying a single file; use dst path provided by user as destination
-          // rather than destination directory
-          absdst = absdst.getParent();
+          // rather than destination directory, if a file
+          Path dstparent = absdst.getParent();
+          if (!(destFileSys.exists(dstparent) &&
+                destFileSys.getFileStatus(dstparent).isDir())) {
+            absdst = dstparent;
+          }
+        }
+        if (destFileSys.exists(absdst) &&
+            destFileSys.getFileStatus(absdst).isDir()) {
+          throw new IOException(absdst + " is a directory");
         }
         rename(destFileSys, tmpfile, absdst);
       }

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java Mon Mar 17 15:10:16
2008
@@ -57,7 +57,9 @@
     private long seed = 0L;
 
     MyFile() {
-      int nLevels = gen.nextInt(MAX_LEVELS);
+      this(gen.nextInt(MAX_LEVELS));
+    }
+    MyFile(int nLevels) {
       String xname = "";
       if (nLevels != 0) {
         int[] levels = new int[nLevels];
@@ -101,8 +103,9 @@
     return files;
   }
 
-  static MyFile createFile(Path root, FileSystem fs) throws IOException {
-    MyFile f = new MyFile();
+  static MyFile createFile(Path root, FileSystem fs, int levels)
+      throws IOException {
+    MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
     Path p = new Path(root, f.getName());
     FSDataOutputStream out = fs.create(p);
     byte[] toWrite = new byte[f.getSize()];
@@ -113,6 +116,10 @@
     return f;
   }
 
+  static MyFile createFile(Path root, FileSystem fs) throws IOException {
+    return createFile(root, fs, -1);
+  }
+
   /** check if the files have been copied correctly. */
   private static boolean checkFiles(String fsname, String topdir, MyFile[] files) 
     throws IOException {
@@ -413,6 +420,25 @@
                         "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
       assertTrue("Source and destination directories do not match.",
           checkFiles("local", TEST_ROOT_DIR+"/dest2", files));     
+      //copy single file to existing dir
+      deldir("local", TEST_ROOT_DIR+"/dest2");
+      fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
+      MyFile[] files2 = {createFile(root, fs, 0)};
+      String sname = files2[0].getName();
+      ToolRunner.run(new CopyFiles(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));     
+      updateFiles("local", TEST_ROOT_DIR+"/srcdat", files2, 1);
+      //copy single file to existing dir w/ dst name conflict
+      ToolRunner.run(new CopyFiles(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));     
     }
     finally {
       deldir("local", TEST_ROOT_DIR+"/destdat");



Mime
View raw message