hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r816506 - in /hadoop/mapreduce/trunk: CHANGES.txt src/tools/org/apache/hadoop/tools/DistCp.java
Date Fri, 18 Sep 2009 07:49:32 GMT
Author: cdouglas
Date: Fri Sep 18 07:49:31 2009
New Revision: 816506

URL: http://svn.apache.org/viewvc?rev=816506&view=rev
Log:
MAPREDUCE-664. Display the number of files deleted by distcp when the
-delete option is specified. Contributed by Ravi Gummadi

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=816506&r1=816505&r2=816506&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Sep 18 07:49:31 2009
@@ -372,6 +372,9 @@
     file data to be copied, without actually performing the copy. (Ravi Gummadi
     via cdouglas)
 
+    MAPREDUCE-664. Display the number of files deleted by distcp when the
+    -delete option is specified. (Ravi Gummadi via cdouglas)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=816506&r1=816505&r2=816506&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Fri Sep 18 07:49:31
2009
@@ -1423,8 +1423,10 @@
     checkDuplication(jobfs, dstfilelist, sorted, conf);
 
     if (dststatus != null && args.flags.contains(Options.DELETE)) {
-      deleteNonexisting(dstfs, dststatus, sorted,
+      long deletedPathsCount = deleteNonexisting(dstfs, dststatus, sorted,
           jobfs, jobDirectory, jobConf, conf);
+      LOG.info("deletedPathsFromDestCount(files+directories)=" +
+               deletedPathsCount);
     }
 
     Path tmpDir = new Path(
@@ -1499,8 +1501,13 @@
     }
   }
   
-  /** Delete the dst files/dirs which do not exist in src */
-  static private void deleteNonexisting(
+  /**
+   * Delete the dst files/dirs which do not exist in src
+   * 
+   * @return total count of files and directories deleted from destination
+   * @throws IOException
+   */
+  static private long deleteNonexisting(
       FileSystem dstfs, FileStatus dstroot, Path dstsorted,
       FileSystem jobfs, Path jobdir, JobConf jobconf, Configuration conf
       ) throws IOException {
@@ -1541,6 +1548,7 @@
     //compare lsr list and dst list  
     SequenceFile.Reader lsrin = null;
     SequenceFile.Reader dstin = null;
+    long deletedPathsCount = 0;
     try {
       lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf);
       dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf);
@@ -1568,6 +1576,7 @@
         else {
           //lsrpath does not exist, delete it
           String s = new Path(dstroot.getPath(), lsrpath.toString()).toString();
+          ++deletedPathsCount;
           if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) {
             shellargs[1] = s;
             int r = 0;
@@ -1587,6 +1596,7 @@
       checkAndClose(lsrin);
       checkAndClose(dstin);
     }
+    return deletedPathsCount;
   }
 
   //is x an ancestor path of y?



Mime
View raw message