hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r909783 - in /hadoop/mapreduce/trunk: CHANGES.txt src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java src/tools/org/apache/hadoop/tools/DistCp.java
Date Sat, 13 Feb 2010 10:11:21 GMT
Author: cdouglas
Date: Sat Feb 13 10:11:20 2010
New Revision: 909783

URL: http://svn.apache.org/viewvc?rev=909783&view=rev
Log:
MAPREDUCE-1305. Improve efficiency of distcp -delete. Contributed by Peter Romianowski

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=909783&r1=909782&r2=909783&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sat Feb 13 10:11:20 2010
@@ -326,6 +326,9 @@
     MAPREDUCE-1469. Sqoop should disable speculative execution in export.
     (Aaron Kimball via tomwhite)
 
+    MAPREDUCE-1305. Improve efficiency of distcp -delete. (Peter Romianowski
+    via cdouglas)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java?rev=909783&r1=909782&r2=909783&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java Sat
Feb 13 10:11:20 2010
@@ -951,6 +951,7 @@
   /** test -delete */
   public void testDelete() throws Exception {
     final Configuration conf = new Configuration();
+    conf.setInt("fs.trash.interval", 60);
     MiniDFSCluster cluster = null;
     try {
       cluster = new MiniDFSCluster(conf, 2, true, null);
@@ -1001,6 +1002,12 @@
         dstresults = removePrefix(dstresults, dstrootdir);
         System.out.println("second dstresults=" +  dstresults);
         assertEquals(srcresults, dstresults);
+        // verify that files removed in -delete were moved to the trash
+        // regrettably, this test will break if Trash changes incompatibly
+        assertTrue(fs.exists(new Path(fs.getHomeDirectory(),
+                ".Trash/Current" + dstrootdir + "/foo")));
+        assertTrue(fs.exists(new Path(fs.getHomeDirectory(),
+                ".Trash/Current" + dstrootdir + "/foobar")));
 
         //cleanup
         deldir(fs, dstrootdir);

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=909783&r1=909782&r2=909783&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Sat Feb 13 10:11:20
2010
@@ -48,12 +48,14 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.mapred.FileOutputFormat;
@@ -1571,7 +1573,7 @@
     //write dst lsr results
     final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
     final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf,
-        dstlsr, Text.class, dstroot.getClass(),
+        dstlsr, Text.class, NullWritable.class,
         SequenceFile.CompressionType.NONE);
     try {
       //do lsr to get all file statuses in dstroot
@@ -1581,7 +1583,7 @@
         if (status.isDir()) {
           for(FileStatus child : dstfs.listStatus(status.getPath())) {
             String relative = makeRelative(dstroot.getPath(), child.getPath());
-            writer.append(new Text(relative), child);
+            writer.append(new Text(relative), NullWritable.get());
             lsrstack.push(child);
           }
         }
@@ -1593,7 +1595,7 @@
     //sort lsr results
     final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted");
     SequenceFile.Sorter sorter = new SequenceFile.Sorter(jobfs,
-        new Text.Comparator(), Text.class, FileStatus.class, jobconf);
+        new Text.Comparator(), Text.class, NullWritable.class, jobconf);
     sorter.sort(dstlsr, sortedlsr);
 
     //compare lsr list and dst list  
@@ -1606,16 +1608,15 @@
 
       //compare sorted lsr list and sorted dst list
       final Text lsrpath = new Text();
-      final FileStatus lsrstatus = new FileStatus();
       final Text dstpath = new Text();
       final Text dstfrom = new Text();
-      final FsShell shell = new FsShell(conf);
-      final String[] shellargs = {"-rmr", null};
+      final Trash trash = new Trash(dstfs, conf);
+      Path lastpath = null;
 
       boolean hasnext = dstin.next(dstpath, dstfrom);
-      for(; lsrin.next(lsrpath, lsrstatus); ) {
+      while (lsrin.next(lsrpath, NullWritable.get())) {
         int dst_cmp_lsr = dstpath.compareTo(lsrpath);
-        for(; hasnext && dst_cmp_lsr < 0; ) {
+        while (hasnext && dst_cmp_lsr < 0) {
           hasnext = dstin.next(dstpath, dstfrom);
           dst_cmp_lsr = dstpath.compareTo(lsrpath);
         }
@@ -1623,23 +1624,15 @@
         if (dst_cmp_lsr == 0) {
           //lsrpath exists in dst, skip it
           hasnext = dstin.next(dstpath, dstfrom);
-        }
-        else {
+        } else {
           //lsrpath does not exist, delete it
-          String s = new Path(dstroot.getPath(), lsrpath.toString()).toString();
+          final Path rmpath = new Path(dstroot.getPath(), lsrpath.toString());
           ++deletedPathsCount;
-          if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) {
-            shellargs[1] = s;
-            int r = 0;
-            try {
-               r = shell.run(shellargs);
-            } catch(Exception e) {
-              throw new IOException("Exception from shell.", e);
-            }
-            if (r != 0) {
-              throw new IOException("\"" + shellargs[0] + " " + shellargs[1]
-                  + "\" returns non-zero value " + r);
+          if ((lastpath == null || !isAncestorPath(lastpath, rmpath))) {
+            if (!(trash.moveToTrash(rmpath) || dstfs.delete(rmpath, true))) {
+              throw new IOException("Failed to delete " + rmpath);
             }
+            lastpath = rmpath;
           }
         }
       }
@@ -1651,7 +1644,9 @@
   }
 
   //is x an ancestor path of y?
-  static private boolean isAncestorPath(String x, String y) {
+  static private boolean isAncestorPath(Path xp, Path yp) {
+    final String x = xp.toString();
+    final String y = yp.toString();
     if (!y.startsWith(x)) {
       return false;
     }



Mime
View raw message