hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r651455 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/util/CopyFiles.java
Date Thu, 24 Apr 2008 23:05:13 GMT
Author: cdouglas
Date: Thu Apr 24 16:05:08 2008
New Revision: 651455

URL: http://svn.apache.org/viewvc?rev=651455&view=rev
Log:
HADOOP-3294. Fix distcp to check the destination length and retry the copy if
it doesn't match the src length. Contributed by Tsz Wo (Nicholas), SZE.


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=651455&r1=651454&r2=651455&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Apr 24 16:05:08 2008
@@ -660,6 +660,9 @@
     HADOOP-3285. Fix input split locality when the splits align to
     fs blocks. (omalley)
 
+    HADOOP-3294. Fix distcp to check the destination length and retry the copy
+    if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via cdouglas)
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=651455&r1=651454&r2=651455&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Thu Apr 24 16:05:08 2008
@@ -382,13 +382,11 @@
         checkAndClose(out);
       }
 
-      final boolean success = cbcopied == srcstat.getLen();
-      if (!success) {
-        final String badlen = "ERROR? copied " + bytesString(cbcopied)
-            + " but expected " + bytesString(srcstat.getLen()) 
-            + " from " + srcstat.getPath();
-        LOG.warn(badlen);
-        outc.collect(null, new Text(badlen));
+      if (cbcopied != srcstat.getLen()) {
+        throw new IOException("File size not matched: copied "
+            + bytesString(cbcopied) + " to tmpfile (=" + tmpfile
+            + ") but expected " + bytesString(srcstat.getLen()) 
+            + " from " + srcstat.getPath());        
       }
       else {
         if (totfiles == 1) {
@@ -408,7 +406,16 @@
           throw new IOException("Failed to craete parent dir: " + absdst.getParent());
         }
         rename(tmpfile, absdst);
-        updatePermissions(srcstat, destFileSys.getFileStatus(absdst));
+
+        FileStatus dststat = destFileSys.getFileStatus(absdst);
+        if (dststat.getLen() != srcstat.getLen()) {
+          destFileSys.delete(absdst, false);
+          throw new IOException("File size not matched: copied "
+              + bytesString(dststat.getLen()) + " to dst (=" + absdst
+              + ") but expected " + bytesString(srcstat.getLen()) 
+              + " from " + srcstat.getPath());        
+        } 
+        updatePermissions(srcstat, dststat);
       }
 
       // report at least once for each file



Mime
View raw message