hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject git commit: HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw)
Date Wed, 24 Sep 2014 22:40:09 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6e5194099 -> ea1fb1461


HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea1fb146
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea1fb146
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea1fb146

Branch: refs/heads/branch-2
Commit: ea1fb1461abb3e3bab727774955bb4a5041c6c15
Parents: 6e51940
Author: Allen Wittenauer <aw@apache.org>
Authored: Wed Sep 24 15:39:55 2014 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Wed Sep 24 15:39:55 2014 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |    2 +
 .../hadoop/tools/CopyListingFileStatus.java     |    2 +-
 .../apache/hadoop/tools/DistCpOptionSwitch.java |   21 +-
 .../org/apache/hadoop/tools/DistCpOptions.java  |    2 +-
 .../apache/hadoop/tools/util/DistCpUtils.java   |   58 +-
 .../apache/hadoop/tools/TestOptionsParser.java  |    2 +-
 .../hadoop/tools/mapred/TestCopyMapper.java     |    1 +
 .../hadoop/tools/util/TestDistCpUtils.java      | 1024 ++++++++++++++++--
 8 files changed, 996 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1fbb90e..c899c42 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -204,6 +204,8 @@ Release 2.6.0 - UNRELEASED
     HADOOP-11017. KMS delegation token secret manager should be able to use 
     zookeeper as store. (asuresh via tucu)
 
+    HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw)
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 04437d7..9b31dfb 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -101,7 +101,7 @@ public final class CopyListingFileStatus extends FileStatus {
    * @return Map<String, byte[]> containing all xAttrs
    */
   public Map<String, byte[]> getXAttrs() {
-    return xAttrs;
+    return xAttrs != null ? xAttrs : Collections.<String, byte[]>emptyMap();
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index c544813..9b85997 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -37,18 +37,21 @@ public enum DistCpOptionSwitch {
   /**
    * Preserves status of file/path in the target.
    * Default behavior with -p, is to preserve replication,
-   * block size, user, group, permission and checksum type on the target file.
-   * Note that when preserving checksum type, block size is also preserved.
+   * block size, user, group, permission, checksum type and timestamps on the 
+   * target file. Note that when preserving checksum type, block size is also 
+   * preserved.
    *
-   * If any of the optional switches are present among rbugpc, then
-   * only the corresponding file attribute is preserved.
+   * @see PRESERVE_STATUS_DEFAULT
    *
+   * If any of the optional switches are present among rbugpcaxt, then
+   * only the corresponding file attribute is preserved.
    */
   PRESERVE_STATUS(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
-      new Option("p", true, "preserve status (rbugpcax)(replication, " +
-          "block-size, user, group, permission, checksum-type, ACL, XATTR).  " +
-          "If -p is specified with no <arg>, then preserves replication, " +
-          "block size, user, group, permission and checksum type." +
+      new Option("p", true, "preserve status (rbugpcaxt)(replication, " +
+          "block-size, user, group, permission, checksum-type, ACL, XATTR, " +
+          "timestamps). If -p is specified with no <arg>, then preserves " +
+          "replication, block size, user, group, permission, checksum type " +
+          "and timestamps. " +
           "raw.* xattrs are preserved when both the source and destination " +
           "paths are in the /.reserved/raw hierarchy (HDFS only). raw.* xattr" +
           "preservation is independent of the -p flag." +
@@ -166,7 +169,7 @@ public enum DistCpOptionSwitch {
   BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
       new Option("bandwidth", true, "Specify bandwidth per map in MB"));
 
-  static final String PRESERVE_STATUS_DEFAULT = "-prbugpc";
+  public static final String PRESERVE_STATUS_DEFAULT = "-prbugpct";
   private final String confLabel;
   private final Option option;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index a2a5be3..57d2fb7 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -68,7 +68,7 @@ public class DistCpOptions {
   private boolean targetPathExists = true;
   
   public static enum FileAttribute{
-    REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR;
+    REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES;
 
     public static FileAttribute getAttribute(char symbol) {
       for (FileAttribute attribute : values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index abd30ee..71e84a1 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -18,39 +18,39 @@
 
 package org.apache.hadoop.tools.util;
 
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.text.DecimalFormat;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclUtil;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException;
 import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException;
 import org.apache.hadoop.tools.CopyListingFileStatus;
+import org.apache.hadoop.tools.DistCpOptions;
 import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
 import org.apache.hadoop.tools.mapred.UniformSizeInputFormat;
-import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException;
-import org.apache.hadoop.tools.DistCpOptions;
-import org.apache.hadoop.mapreduce.InputFormat;
 
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.text.DecimalFormat;
-import java.net.URI;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
+import com.google.common.collect.Maps;
 
 /**
  * Utility functions used in DistCp.
@@ -163,7 +163,7 @@ public class DistCpUtils {
   }
 
   /**
-   * Un packs preservation attribute string containing the first character of
+   * Unpacks preservation attribute string containing the first character of
    * each preservation attribute back to a set of attributes to preserve
    * @param attributes - Attribute string
    * @return - Attribute set
@@ -209,7 +209,7 @@ public class DistCpUtils {
       if (!srcAcl.equals(targetAcl)) {
         targetFS.setAcl(path, srcAcl);
       }
-      // setAcl can't preserve sticky bit, so also call setPermission if needed.
+      // setAcl doesn't preserve sticky bit, so also call setPermission if needed.
       if (srcFileStatus.getPermission().getStickyBit() !=
           targetFileStatus.getPermission().getStickyBit()) {
         targetFS.setPermission(path, srcFileStatus.getPermission());
@@ -225,30 +225,28 @@ public class DistCpUtils {
       Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs();
       Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path);
       if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) {
-        Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
-        while (iter.hasNext()) {
-          Entry<String, byte[]> entry = iter.next();
-          final String xattrName = entry.getKey();
+        for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) {
+          String xattrName = entry.getKey();
           if (xattrName.startsWith(rawNS) || preserveXAttrs) {
-            targetFS.setXAttr(path, entry.getKey(), entry.getValue());
+            targetFS.setXAttr(path, xattrName, entry.getValue());
           }
         }
       }
     }
 
-    if (attributes.contains(FileAttribute.REPLICATION) && ! targetFileStatus.isDirectory() &&
-        srcFileStatus.getReplication() != targetFileStatus.getReplication()) {
+    if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() &&
+        (srcFileStatus.getReplication() != targetFileStatus.getReplication())) {
       targetFS.setReplication(path, srcFileStatus.getReplication());
     }
 
     if (attributes.contains(FileAttribute.GROUP) &&
-            !group.equals(srcFileStatus.getGroup())) {
+        !group.equals(srcFileStatus.getGroup())) {
       group = srcFileStatus.getGroup();
       chown = true;
     }
 
     if (attributes.contains(FileAttribute.USER) &&
-            !user.equals(srcFileStatus.getOwner())) {
+        !user.equals(srcFileStatus.getOwner())) {
       user = srcFileStatus.getOwner();
       chown = true;
     }
@@ -256,6 +254,12 @@ public class DistCpUtils {
     if (chown) {
       targetFS.setOwner(path, user, group);
     }
+    
+    if (attributes.contains(FileAttribute.TIMES)) {
+      targetFS.setTimes(path, 
+          srcFileStatus.getModificationTime(), 
+          srcFileStatus.getAccessTime());
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index 2c42269..30fb25b 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -497,7 +497,7 @@ public class TestOptionsParser {
       attribIterator.next();
       i++;
     }
-    Assert.assertEquals(i, 6);
+    Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2);
 
     try {
       OptionsParser.parse(new String[] {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
index 88540a0..7639d23 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
@@ -590,6 +590,7 @@ public class TestCopyMapper {
           EnumSet.allOf(DistCpOptions.FileAttribute.class);
       preserveStatus.remove(DistCpOptions.FileAttribute.ACL);
       preserveStatus.remove(DistCpOptions.FileAttribute.XATTR);
+      preserveStatus.remove(DistCpOptions.FileAttribute.TIMES);
 
       context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
         DistCpUtils.packAttributes(preserveStatus));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1fb146/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
index c4b64b9..200d786 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
@@ -18,38 +18,44 @@
 
 package org.apache.hadoop.tools.util;
 
-import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.EnumSet;
+import java.util.Random;
+import java.util.Stack;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.tools.CopyListingFileStatus;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Test;
 import org.junit.BeforeClass;
-import org.junit.AfterClass;
-
-import java.util.EnumSet;
-import java.util.Random;
-import java.util.Stack;
-import java.io.IOException;
-import java.io.OutputStream;
+import org.junit.Test;
 
 public class TestDistCpUtils {
   private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class);
 
   private static final Configuration config = new Configuration();
   private static MiniDFSCluster cluster;
-
+  private static final FsPermission fullPerm = new FsPermission((short) 777);
+  private static final FsPermission almostFullPerm = new FsPermission((short) 666);
+  private static final FsPermission noPerm = new FsPermission((short) 0);
+  
   @BeforeClass
   public static void create() throws IOException {
-    cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).format(true)
-                                                .build(); 
+    cluster = new MiniDFSCluster.Builder(config)
+        .numDataNodes(1)
+        .format(true)
+        .build(); 
   }
 
   @AfterClass
@@ -61,13 +67,16 @@ public class TestDistCpUtils {
 
   @Test
   public void testGetRelativePathRoot() {
+    Path root = new Path("/");
+    Path child = new Path("/a");
+    Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a");
+  }
+
+  @Test
+  public void testGetRelativePath() {
     Path root = new Path("/tmp/abc");
     Path child = new Path("/tmp/abc/xyz/file");
     Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/xyz/file");
-
-    root = new Path("/");
-    child = new Path("/a");
-    Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a");
   }
 
   @Test
@@ -77,70 +86,916 @@ public class TestDistCpUtils {
 
     attributes.add(FileAttribute.REPLICATION);
     Assert.assertEquals(DistCpUtils.packAttributes(attributes), "R");
-    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("R"));
 
     attributes.add(FileAttribute.BLOCKSIZE);
     Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RB");
-    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RB"));
 
     attributes.add(FileAttribute.USER);
-    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBU");
-    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBU"));
+    attributes.add(FileAttribute.CHECKSUMTYPE);
+    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUC");
 
     attributes.add(FileAttribute.GROUP);
-    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUG");
-    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUG"));
+    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGC");
 
     attributes.add(FileAttribute.PERMISSION);
-    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGP");
-    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUGP"));
+    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPC");
+
+    attributes.add(FileAttribute.TIMES);
+    Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPCT");
+  }
+
+  public void testUnpackAttributes() {
+    EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RCBUGPAXT"));
+
+    attributes.remove(FileAttribute.REPLICATION);
+    attributes.remove(FileAttribute.CHECKSUMTYPE);
+    attributes.remove(FileAttribute.ACL);
+    attributes.remove(FileAttribute.XATTR);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGPT"));
+
+    attributes.remove(FileAttribute.TIMES);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGP"));
+
+    attributes.remove(FileAttribute.BLOCKSIZE);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UGP"));
+
+    attributes.remove(FileAttribute.GROUP);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UP"));
+
+    attributes.remove(FileAttribute.USER);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("P"));
+
+    attributes.remove(FileAttribute.PERMISSION);
+    Assert.assertEquals(attributes, DistCpUtils.unpackAttributes(""));
   }
 
   @Test
-  public void testPreserve() {
-    try {
-      FileSystem fs = FileSystem.get(config);
-      EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
-
-
-      Path path = new Path("/tmp/abc");
-      Path src = new Path("/tmp/src");
-      fs.mkdirs(path);
-      fs.mkdirs(src);
-      CopyListingFileStatus srcStatus = new CopyListingFileStatus(
-        fs.getFileStatus(src));
-
-      FsPermission noPerm = new FsPermission((short) 0);
-      fs.setPermission(path, noPerm);
-      fs.setOwner(path, "nobody", "nobody");
-
-      DistCpUtils.preserve(fs, path, srcStatus, attributes, false);
-      FileStatus target = fs.getFileStatus(path);
-      Assert.assertEquals(target.getPermission(), noPerm);
-      Assert.assertEquals(target.getOwner(), "nobody");
-      Assert.assertEquals(target.getGroup(), "nobody");
-
-      attributes.add(FileAttribute.PERMISSION);
-      DistCpUtils.preserve(fs, path, srcStatus, attributes, false);
-      target = fs.getFileStatus(path);
-      Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
-      Assert.assertEquals(target.getOwner(), "nobody");
-      Assert.assertEquals(target.getGroup(), "nobody");
-
-      attributes.add(FileAttribute.GROUP);
-      attributes.add(FileAttribute.USER);
-      DistCpUtils.preserve(fs, path, srcStatus, attributes, false);
-      target = fs.getFileStatus(path);
-      Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
-      Assert.assertEquals(target.getOwner(), srcStatus.getOwner());
-      Assert.assertEquals(target.getGroup(), srcStatus.getGroup());
-
-      fs.delete(path, true);
-      fs.delete(src, true);
-    } catch (IOException e) {
-      LOG.error("Exception encountered ", e);
-      Assert.fail("Preserve test failure");
-    }
+  public void testPreserveDefaults() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    
+    // preserve replication, block size, user, group, permission, 
+    // checksum type and timestamps    
+    EnumSet<FileAttribute> attributes = 
+        DistCpUtils.unpackAttributes(
+            DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.substring(1));
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+    
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+  
+  @Test
+  public void testPreserveNothingOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertTrue(dstStatus.getAccessTime() == 100);
+    Assert.assertTrue(dstStatus.getModificationTime() == 100);
+    Assert.assertTrue(dstStatus.getReplication() == 0);
+  }
+
+  @Test
+  public void testPreservePermissionOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+  }
+
+  @Test
+  public void testPreserveGroupOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
+  }
+
+  @Test
+  public void testPreserveUserOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+  }
+
+  @Test
+  public void testPreserveReplicationOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    // Replication shouldn't apply to dirs so this should still be 0 == 0
+    Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveTimestampOnDirectory() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);
+
+    Path dst = new Path("/tmp/abc");
+    Path src = new Path("/tmp/src");
+
+    createDirectory(fs, src);
+    createDirectory(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+  }
+
+  @Test
+  public void testPreserveNothingOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreservePermissionOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveGroupOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveUserOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveReplicationOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveTimestampOnFile() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);
+
+    Path dst = new Path("/tmp/dest2");
+    Path src = new Path("/tmp/src2");
+
+    createFile(fs, src);
+    createFile(fs, dst);
+
+    fs.setPermission(src, fullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(dst, noPerm);
+    fs.setOwner(dst, "nobody", "nobody-group");
+    fs.setTimes(dst, 100, 100);
+    fs.setReplication(dst, (short) 2);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);
+
+    CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
+  }
+
+  @Test
+  public void testPreserveOnFileUpwardRecursion() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
+    // Remove ACL because tests run with dfs.namenode.acls.enabled false
+    attributes.remove(FileAttribute.ACL);
+    
+    Path src = new Path("/tmp/src2");
+    Path f0 = new Path("/f0");
+    Path f1 = new Path("/d1/f1");
+    Path f2 = new Path("/d1/d2/f2");
+    Path d1 = new Path("/d1/");
+    Path d2 = new Path("/d1/d2/");
+
+    createFile(fs, src);
+    createFile(fs, f0);
+    createFile(fs, f1);
+    createFile(fs, f2);
+
+    fs.setPermission(src, almostFullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(d1, fullPerm);
+    fs.setOwner(d1, "anybody", "anybody-group");
+    fs.setTimes(d1, 400, 400);
+    fs.setReplication(d1, (short) 3);
+
+    fs.setPermission(d2, fullPerm);
+    fs.setOwner(d2, "anybody", "anybody-group");
+    fs.setTimes(d2, 300, 300);
+    fs.setReplication(d2, (short) 3);
+
+    fs.setPermission(f0, fullPerm);
+    fs.setOwner(f0, "anybody", "anybody-group");
+    fs.setTimes(f0, 200, 200);
+    fs.setReplication(f0, (short) 3);
+
+    fs.setPermission(f1, fullPerm);
+    fs.setOwner(f1, "anybody", "anybody-group");
+    fs.setTimes(f1, 200, 200);
+    fs.setReplication(f1, (short) 3);
+
+    fs.setPermission(f2, fullPerm);
+    fs.setOwner(f2, "anybody", "anybody-group");
+    fs.setTimes(f2, 200, 200);
+    fs.setReplication(f2, (short) 3);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, f2, srcStatus, attributes, false);
+
+    cluster.triggerHeartbeats();
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    // attributes of src -> f2 ? should be yes
+    CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2));
+    Assert.assertTrue(srcStatus.getPermission().equals(f2Status.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(f2Status.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(f2Status.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == f2Status.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == f2Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() == f2Status.getReplication());
+
+    // attributes of src -> f1 ? should be no
+    CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1));
+    Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication());
+
+    // attributes of src -> f0 ? should be no
+    CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0));
+    Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication());
+
+    // attributes of src -> d2 ? should be no
+    CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2));
+    Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup()));
+    Assert.assertTrue(d2Status.getAccessTime() == 300);
+    Assert.assertTrue(d2Status.getModificationTime() == 300);
+    Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication());
+
+    // attributes of src -> d1 ? should be no
+    CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1));
+    Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup()));
+    Assert.assertTrue(d1Status.getAccessTime() == 400);
+    Assert.assertTrue(d1Status.getModificationTime() == 400);
+    Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication());
+  }
+
+  @Test
+  public void testPreserveOnDirectoryUpwardRecursion() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
+    
+    // Remove ACL because tests run with dfs.namenode.acls.enabled false
+    attributes.remove(FileAttribute.ACL);
+
+    Path src = new Path("/tmp/src2");
+    Path f0 = new Path("/f0");
+    Path f1 = new Path("/d1/f1");
+    Path f2 = new Path("/d1/d2/f2");
+    Path d1 = new Path("/d1/");
+    Path d2 = new Path("/d1/d2/");
+
+    createFile(fs, src);
+    createFile(fs, f0);
+    createFile(fs, f1);
+    createFile(fs, f2);
+
+    fs.setPermission(src, almostFullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(d1, fullPerm);
+    fs.setOwner(d1, "anybody", "anybody-group");
+    fs.setTimes(d1, 400, 400);
+    fs.setReplication(d1, (short) 3);
+
+    fs.setPermission(d2, fullPerm);
+    fs.setOwner(d2, "anybody", "anybody-group");
+    fs.setTimes(d2, 300, 300);
+    fs.setReplication(d2, (short) 3);
+
+    fs.setPermission(f0, fullPerm);
+    fs.setOwner(f0, "anybody", "anybody-group");
+    fs.setTimes(f0, 200, 200);
+    fs.setReplication(f0, (short) 3);
+
+    fs.setPermission(f1, fullPerm);
+    fs.setOwner(f1, "anybody", "anybody-group");
+    fs.setTimes(f1, 200, 200);
+    fs.setReplication(f1, (short) 3);
+
+    fs.setPermission(f2, fullPerm);
+    fs.setOwner(f2, "anybody", "anybody-group");
+    fs.setTimes(f2, 200, 200);
+    fs.setReplication(f2, (short) 3);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, d2, srcStatus, attributes, false);
+
+    cluster.triggerHeartbeats();
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    // attributes of src -> d2 ? should be yes
+    CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2));
+    Assert.assertTrue(srcStatus.getPermission().equals(d2Status.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(d2Status.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(d2Status.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == d2Status.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == d2Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication());
+
+    // attributes of src -> d1 ? should be no
+    CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1));
+    Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication());
+
+    // attributes of src -> f2 ? should be no
+    CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2));
+    Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication());
+
+    // attributes of src -> f1 ? should be no
+    CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1));
+    Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication());
+
+    // attributes of src -> f0 ? should be no
+    CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0));
+    Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication());
+  }
+
+  @Test
+  public void testPreserveOnFileDownwardRecursion() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
+    // Remove ACL because tests run with dfs.namenode.acls.enabled false
+    attributes.remove(FileAttribute.ACL);
+
+    Path src = new Path("/tmp/src2");
+    Path f0 = new Path("/f0");
+    Path f1 = new Path("/d1/f1");
+    Path f2 = new Path("/d1/d2/f2");
+    Path d1 = new Path("/d1/");
+    Path d2 = new Path("/d1/d2/");
+
+    createFile(fs, src);
+    createFile(fs, f0);
+    createFile(fs, f1);
+    createFile(fs, f2);
+
+    fs.setPermission(src, almostFullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(d1, fullPerm);
+    fs.setOwner(d1, "anybody", "anybody-group");
+    fs.setTimes(d1, 400, 400);
+    fs.setReplication(d1, (short) 3);
+
+    fs.setPermission(d2, fullPerm);
+    fs.setOwner(d2, "anybody", "anybody-group");
+    fs.setTimes(d2, 300, 300);
+    fs.setReplication(d2, (short) 3);
+
+    fs.setPermission(f0, fullPerm);
+    fs.setOwner(f0, "anybody", "anybody-group");
+    fs.setTimes(f0, 200, 200);
+    fs.setReplication(f0, (short) 3);
+
+    fs.setPermission(f1, fullPerm);
+    fs.setOwner(f1, "anybody", "anybody-group");
+    fs.setTimes(f1, 200, 200);
+    fs.setReplication(f1, (short) 3);
+
+    fs.setPermission(f2, fullPerm);
+    fs.setOwner(f2, "anybody", "anybody-group");
+    fs.setTimes(f2, 200, 200);
+    fs.setReplication(f2, (short) 3);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, f0, srcStatus, attributes, false);
+
+    cluster.triggerHeartbeats();
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    // attributes of src -> f0 ? should be yes
+    CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0));
+    Assert.assertTrue(srcStatus.getPermission().equals(f0Status.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(f0Status.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(f0Status.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == f0Status.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == f0Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() == f0Status.getReplication());
+
+    // attributes of src -> f1 ? should be no
+    CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1));
+    Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication());
+
+    // attributes of src -> f2 ? should be no
+    CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2));
+    Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication());
+
+    // attributes of src -> d1 ? should be no
+    CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1));
+    Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup()));
+    Assert.assertTrue(d1Status.getAccessTime() == 400);
+    Assert.assertTrue(d1Status.getModificationTime() == 400);
+    Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication());
+
+    // attributes of src -> d2 ? should be no
+    CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2));
+    Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup()));
+    Assert.assertTrue(d2Status.getAccessTime() == 300);
+    Assert.assertTrue(d2Status.getModificationTime() == 300);
+    Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication());
+  }
+
+  @Test
+  public void testPreserveOnDirectoryDownwardRecursion() throws IOException {
+    FileSystem fs = FileSystem.get(config);
+    EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
+    // Remove ACL because tests run with dfs.namenode.acls.enabled false
+    attributes.remove(FileAttribute.ACL);
+
+    Path src = new Path("/tmp/src2");
+    Path f0 = new Path("/f0");
+    Path f1 = new Path("/d1/f1");
+    Path f2 = new Path("/d1/d2/f2");
+    Path d1 = new Path("/d1/");
+    Path d2 = new Path("/d1/d2/");
+    Path root = new Path("/");
+
+    createFile(fs, src);
+    createFile(fs, f0);
+    createFile(fs, f1);
+    createFile(fs, f2);
+
+    fs.setPermission(src, almostFullPerm);
+    fs.setOwner(src, "somebody", "somebody-group");
+    fs.setTimes(src, 0, 0);
+    fs.setReplication(src, (short) 1);
+
+    fs.setPermission(root, fullPerm);
+    fs.setOwner(root, "anybody", "anybody-group");
+    fs.setTimes(root, 400, 400);
+    fs.setReplication(root, (short) 3);
+
+    fs.setPermission(d1, fullPerm);
+    fs.setOwner(d1, "anybody", "anybody-group");
+    fs.setTimes(d1, 400, 400);
+    fs.setReplication(d1, (short) 3);
+
+    fs.setPermission(d2, fullPerm);
+    fs.setOwner(d2, "anybody", "anybody-group");
+    fs.setTimes(d2, 300, 300);
+    fs.setReplication(d2, (short) 3);
+
+    fs.setPermission(f0, fullPerm);
+    fs.setOwner(f0, "anybody", "anybody-group");
+    fs.setTimes(f0, 200, 200);
+    fs.setReplication(f0, (short) 3);
+
+    fs.setPermission(f1, fullPerm);
+    fs.setOwner(f1, "anybody", "anybody-group");
+    fs.setTimes(f1, 200, 200);
+    fs.setReplication(f1, (short) 3);
+
+    fs.setPermission(f2, fullPerm);
+    fs.setOwner(f2, "anybody", "anybody-group");
+    fs.setTimes(f2, 200, 200);
+    fs.setReplication(f2, (short) 3);
+
+    CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));
+
+    DistCpUtils.preserve(fs, root, srcStatus, attributes, false);
+
+    cluster.triggerHeartbeats();
+
+    // FileStatus.equals only compares path field, must explicitly compare all fields
+    // attributes of src -> root ? should be yes
+    CopyListingFileStatus rootStatus = new CopyListingFileStatus(fs.getFileStatus(root));
+    Assert.assertTrue(srcStatus.getPermission().equals(rootStatus.getPermission()));
+    Assert.assertTrue(srcStatus.getOwner().equals(rootStatus.getOwner()));
+    Assert.assertTrue(srcStatus.getGroup().equals(rootStatus.getGroup()));
+    Assert.assertTrue(srcStatus.getAccessTime() == rootStatus.getAccessTime());
+    Assert.assertTrue(srcStatus.getModificationTime() == rootStatus.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() != rootStatus.getReplication());
+
+    // attributes of src -> d1 ? should be no
+    CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1));
+    Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication());
+
+    // attributes of src -> d2 ? should be no
+    CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2));
+    Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == d2Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == d2Status.getModificationTime());
+    Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication());
+
+    // attributes of src -> f0 ? should be no
+    CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0));
+    Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication());
+
+    // attributes of src -> f1 ? should be no
+    CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1));
+    Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication());
+
+    // attributes of src -> f2 ? should be no
+    CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2));
+    Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission()));
+    Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner()));
+    Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup()));
+    Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime());
+    Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime());
+    Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication());
   }
 
   private static Random rand = new Random();
@@ -168,10 +1023,10 @@ public class TestDistCpUtils {
     fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm);
     fs.setPermission(new Path(base + "/newTest/hello/world3"), perm);
     fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm);
-    createFile(fs, base + "/newTest/1");
-    createFile(fs, base + "/newTest/hello/2");
-    createFile(fs, base + "/newTest/hello/world3/oldworld/3");
-    createFile(fs, base + "/newTest/hello/world2/4");
+    createFile(fs, new Path(base, "/newTest/1"));
+    createFile(fs, new Path(base, "/newTest/hello/2"));
+    createFile(fs, new Path(base, "/newTest/hello/world3/oldworld/3"));
+    createFile(fs, new Path(base, "/newTest/hello/world2/4"));
     return base;
   }
 
@@ -191,12 +1046,27 @@ public class TestDistCpUtils {
       LOG.warn("Exception encountered ", e);
     }
   }
-
+  
   public static void createFile(FileSystem fs, String filePath) throws IOException {
-    OutputStream out = fs.create(new Path(filePath));
+    Path path = new Path(filePath);
+    createFile(fs, path);
+  }
+
+  /** Creates a new, empty file at filePath and always overwrites */
+  public static void createFile(FileSystem fs, Path filePath) throws IOException {
+    OutputStream out = fs.create(filePath, true);
     IOUtils.closeStream(out);
   }
 
+  /** Creates a new, empty directory at dirPath and always overwrites */
+  public static void createDirectory(FileSystem fs, Path dirPath) throws IOException {
+    fs.delete(dirPath, true);
+    boolean created = fs.mkdirs(dirPath);
+    if (!created) {
+      LOG.warn("Could not create directory " + dirPath + " this might cause test failures.");
+    }
+  }
+
   public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase)
       throws IOException {
     Path base = new Path(targetBase);


Mime
View raw message