hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From umamah...@apache.org
Subject svn commit: r1602116 - in /hadoop/common/branches/branch-2: ./ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/ hadoop-tools/hadoop-distcp/src/main/java/org/apach...
Date Thu, 12 Jun 2014 10:59:28 GMT
Author: umamahesh
Date: Thu Jun 12 10:59:28 2014
New Revision: 1602116

URL: http://svn.apache.org/r1602116
Log:
Merged from trunk to branch-2. MAPREDUCE-5898. distcp to support preserving HDFS extended
attributes(XAttrs). Contributed by Yi Liu. 

Added:
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithXAttrs.java
      - copied unchanged from r1600900, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithXAttrs.java
Modified:
    hadoop/common/branches/branch-2/   (props changed)
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java

Propchange: hadoop/common/branches/branch-2/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk:r1600900

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java
Thu Jun 12 10:59:28 2014
@@ -129,6 +129,7 @@ public abstract class CopyListing extend
   /**
    * Validate the final resulting path listing.  Checks if there are duplicate
    * entries.  If preserving ACLs, checks that file system can support ACLs.
+   * If preserving XAttrs, checks that file system can support XAttrs.
    *
    * @param pathToListFile - path listing build by doBuildListing
    * @param options - Input options to distcp
@@ -151,6 +152,7 @@ public abstract class CopyListing extend
 
       Text currentKey = new Text();
       Set<URI> aclSupportCheckFsSet = Sets.newHashSet();
+      Set<URI> xAttrSupportCheckFsSet = Sets.newHashSet();
       while (reader.next(currentKey)) {
         if (currentKey.equals(lastKey)) {
           CopyListingFileStatus currentFileStatus = new CopyListingFileStatus();
@@ -167,6 +169,14 @@ public abstract class CopyListing extend
             aclSupportCheckFsSet.add(lastFsUri);
           }
         }
+        if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
+          FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config);
+          URI lastFsUri = lastFs.getUri();
+          if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
+            DistCpUtils.checkFileSystemXAttrSupport(lastFs);
+            xAttrSupportCheckFsSet.add(lastFsUri);
+          }
+        }
         lastKey.set(currentKey);
       }
     } finally {
@@ -256,4 +266,10 @@ public abstract class CopyListing extend
       super(message);
     }
   }
+  
+  public static class XAttrsNotSupportedException extends RuntimeException {
+    public XAttrsNotSupportedException(String message) {
+      super(message);
+    }
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
Thu Jun 12 10:59:28 2014
@@ -21,7 +21,10 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.FileStatus;
@@ -34,6 +37,7 @@ import org.apache.hadoop.io.WritableUtil
 
 import com.google.common.base.Objects;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 /**
  * CopyListingFileStatus is a specialized subclass of {@link FileStatus} for
@@ -45,6 +49,7 @@ import com.google.common.collect.Lists;
 public final class CopyListingFileStatus extends FileStatus {
 
   private static final byte NO_ACL_ENTRIES = -1;
+  private static final int NO_XATTRS = -1;
 
   // Retain static arrays of enum values to prevent repeated allocation of new
   // arrays during deserialization.
@@ -53,6 +58,7 @@ public final class CopyListingFileStatus
   private static final FsAction[] FS_ACTIONS = FsAction.values();
 
   private List<AclEntry> aclEntries;
+  private Map<String, byte[]> xAttrs;
 
   /**
    * Default constructor.
@@ -88,6 +94,24 @@ public final class CopyListingFileStatus
   public void setAclEntries(List<AclEntry> aclEntries) {
     this.aclEntries = aclEntries;
   }
+  
+  /**
+   * Returns all xAttrs.
+   * 
+   * @return Map<String, byte[]> containing all xAttrs
+   */
+  public Map<String, byte[]> getXAttrs() {
+    return xAttrs;
+  }
+  
+  /**
+   * Sets optional xAttrs.
+   * 
+   * @param xAttrs Map<String, byte[]> containing all xAttrs
+   */
+  public void setXAttrs(Map<String, byte[]> xAttrs) {
+    this.xAttrs = xAttrs;
+  }
 
   @Override
   public void write(DataOutput out) throws IOException {
@@ -104,6 +128,26 @@ public final class CopyListingFileStatus
     } else {
       out.writeByte(NO_ACL_ENTRIES);
     }
+    
+    if (xAttrs != null) {
+      out.writeInt(xAttrs.size());
+      Iterator<Entry<String, byte[]>> iter = xAttrs.entrySet().iterator();
+      while (iter.hasNext()) {
+        Entry<String, byte[]> entry = iter.next();
+        WritableUtils.writeString(out, entry.getKey());
+        final byte[] value = entry.getValue();
+        if (value != null) {
+          out.writeInt(value.length);
+          if (value.length > 0) {
+            out.write(value);
+          }
+        } else {
+          out.writeInt(-1);
+        }
+      }
+    } else {
+      out.writeInt(NO_XATTRS);
+    }
   }
 
   @Override
@@ -123,6 +167,25 @@ public final class CopyListingFileStatus
     } else {
       aclEntries = null;
     }
+    
+    int xAttrsSize = in.readInt();
+    if (xAttrsSize != NO_XATTRS) {
+      xAttrs = Maps.newHashMap();
+      for (int i = 0; i < xAttrsSize; ++i) {
+        final String name = WritableUtils.readString(in);
+        final int valueLen = in.readInt();
+        byte[] value = null;
+        if (valueLen > -1) {
+          value = new byte[valueLen];
+          if (valueLen > 0) {
+            in.readFully(value);
+          }
+        }
+        xAttrs.put(name, value);
+      }
+    } else {
+      xAttrs = null;
+    }
   }
 
   @Override
@@ -134,12 +197,13 @@ public final class CopyListingFileStatus
       return false;
     }
     CopyListingFileStatus other = (CopyListingFileStatus)o;
-    return Objects.equal(aclEntries, other.aclEntries);
+    return Objects.equal(aclEntries, other.aclEntries) &&
+        Objects.equal(xAttrs, other.xAttrs);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hashCode(super.hashCode(), aclEntries);
+    return Objects.hashCode(super.hashCode(), aclEntries, xAttrs);
   }
 
   @Override
@@ -147,6 +211,7 @@ public final class CopyListingFileStatus
     StringBuilder sb = new StringBuilder(super.toString());
     sb.append('{');
     sb.append("aclEntries = " + aclEntries);
+    sb.append(", xAttrs = " + xAttrs);
     sb.append('}');
     return sb.toString();
   }

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
Thu Jun 12 10:59:28 2014
@@ -128,6 +128,9 @@ public class DistCp extends Configured i
     } catch (AclsNotSupportedException e) {
       LOG.error("ACLs not supported on at least one file system: ", e);
       return DistCpConstants.ACLS_NOT_SUPPORTED;
+    } catch (XAttrsNotSupportedException e) {
+      LOG.error("XAttrs not supported on at least one file system: ", e);
+      return DistCpConstants.XATTRS_NOT_SUPPORTED;
     } catch (Exception e) {
       LOG.error("Exception encountered ", e);
       return DistCpConstants.UNKNOWN_ERROR;
@@ -304,6 +307,9 @@ public class DistCp extends Configured i
     if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.ACL)) {
       DistCpUtils.checkFileSystemAclSupport(targetFS);
     }
+    if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) {
+      DistCpUtils.checkFileSystemXAttrSupport(targetFS);
+    }
     if (inputOptions.shouldAtomicCommit()) {
       Path workDir = inputOptions.getAtomicWorkPath();
       if (workDir == null) {

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
Thu Jun 12 10:59:28 2014
@@ -117,6 +117,7 @@ public class DistCpConstants {
   public static final int INVALID_ARGUMENT = -1;
   public static final int DUPLICATE_INPUT = -2;
   public static final int ACLS_NOT_SUPPORTED = -3;
+  public static final int XATTRS_NOT_SUPPORTED = -4;
   public static final int UNKNOWN_ERROR = -999;
   
   /**

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
Thu Jun 12 10:59:28 2014
@@ -45,10 +45,10 @@ public enum DistCpOptionSwitch {
    *
    */
   PRESERVE_STATUS(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
-      new Option("p", true, "preserve status (rbugpca)(replication, " +
-          "block-size, user, group, permission, checksum-type, ACL).  If " +
-          "-p is specified with no <arg>, then preserves replication, block " +
-          "size, user, group, permission and checksum type.")),
+      new Option("p", true, "preserve status (rbugpcax)(replication, " +
+          "block-size, user, group, permission, checksum-type, ACL, XATTR).  " +
+          "If -p is specified with no <arg>, then preserves replication, " +
+          "block size, user, group, permission and checksum type.")),
 
   /**
    * Update target location by copying only files that are missing

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
Thu Jun 12 10:59:28 2014
@@ -66,7 +66,7 @@ public class DistCpOptions {
   private boolean targetPathExists = true;
   
   public static enum FileAttribute{
-    REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL;
+    REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR;
 
     public static FileAttribute getAttribute(char symbol) {
       for (FileAttribute attribute : values()) {

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
Thu Jun 12 10:59:28 2014
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.IOUtils;
@@ -36,7 +35,6 @@ import org.apache.hadoop.security.Creden
 import com.google.common.annotations.VisibleForTesting;
 
 import java.io.*;
-import java.util.List;
 import java.util.Stack;
 
 /**
@@ -123,7 +121,7 @@ public class SimpleCopyListing extends C
    *     the the source root is a directory, then the source root entry is not 
    *     written to the sequence file, because only the contents of the source
    *     directory need to be copied in this case.
-   * See {@link org.apache.hadoop.tools.util.DistCpUtils.getRelativePath} for
+   * See {@link org.apache.hadoop.tools.util.DistCpUtils#getRelativePath} for
    *     how relative path is computed.
    * See computeSourceRootPath method for how the root path of the source is
    *     computed.
@@ -147,7 +145,8 @@ public class SimpleCopyListing extends C
         if (!explore || rootStatus.isDirectory()) {
           CopyListingFileStatus rootCopyListingStatus =
             DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus,
-              options.shouldPreserve(FileAttribute.ACL));
+              options.shouldPreserve(FileAttribute.ACL), 
+              options.shouldPreserve(FileAttribute.XATTR));
           writeToFileListingRoot(fileListWriter, rootCopyListingStatus,
               sourcePathRoot, options);
         }
@@ -159,7 +158,8 @@ public class SimpleCopyListing extends C
             CopyListingFileStatus sourceCopyListingStatus =
               DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus,
                 options.shouldPreserve(FileAttribute.ACL) &&
-                sourceStatus.isDirectory());
+                sourceStatus.isDirectory(), options.shouldPreserve(
+                    FileAttribute.XATTR) && sourceStatus.isDirectory());
             writeToFileListing(fileListWriter, sourceCopyListingStatus,
                 sourcePathRoot, options);
 
@@ -271,7 +271,8 @@ public class SimpleCopyListing extends C
                     + sourceStatus.getPath() + " for copy.");
         CopyListingFileStatus childCopyListingStatus =
           DistCpUtils.toCopyListingFileStatus(sourceFS, child,
-            options.shouldPreserve(FileAttribute.ACL) && child.isDirectory());
+            options.shouldPreserve(FileAttribute.ACL) && child.isDirectory(), 
+            options.shouldPreserve(FileAttribute.XATTR) && child.isDirectory());
         writeToFileListing(fileListWriter, childCopyListingStatus,
              sourcePathRoot, options);
         if (isDirectoryAndNotEmpty(sourceFS, child)) {

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
Thu Jun 12 10:59:28 2014
@@ -213,7 +213,8 @@ public class CopyMapper extends Mapper<T
         sourceFS = sourcePath.getFileSystem(conf);
         sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
           sourceFS.getFileStatus(sourcePath),
-          fileAttributes.contains(FileAttribute.ACL));
+          fileAttributes.contains(FileAttribute.ACL), 
+          fileAttributes.contains(FileAttribute.XATTR));
       } catch (FileNotFoundException e) {
         throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
       }

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
Thu Jun 12 10:59:28 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.A
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException;
 import org.apache.hadoop.tools.CopyListingFileStatus;
 import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
 import org.apache.hadoop.tools.mapred.UniformSizeInputFormat;
@@ -39,8 +40,11 @@ import org.apache.hadoop.mapreduce.Input
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.text.DecimalFormat;
 import java.net.URI;
 import java.net.InetAddress;
@@ -210,6 +214,18 @@ public class DistCpUtils {
       !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) {
       targetFS.setPermission(path, srcFileStatus.getPermission());
     }
+    
+    if (attributes.contains(FileAttribute.XATTR)) {
+      Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs();
+      Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path);
+      if (!srcXAttrs.equals(targetXAttrs)) {
+        Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
+        while (iter.hasNext()) {
+          Entry<String, byte[]> entry = iter.next();
+          targetFS.setXAttr(path, entry.getKey(), entry.getValue());
+        }
+      }
+    }
 
     if (attributes.contains(FileAttribute.REPLICATION) && ! targetFileStatus.isDirectory()
&&
         srcFileStatus.getReplication() != targetFileStatus.getReplication()) {
@@ -247,19 +263,34 @@ public class DistCpUtils {
       .getEntries();
     return AclUtil.getAclFromPermAndEntries(fileStatus.getPermission(), entries);
   }
+  
+  /**
+   * Returns a file's all xAttrs.
+   * 
+   * @param fileSystem FileSystem containing the file
+   * @param path file path
+   * @return Map<String, byte[]> containing all xAttrs
+   * @throws IOException if there is an I/O error
+   */
+  public static Map<String, byte[]> getXAttrs(FileSystem fileSystem,
+      Path path) throws IOException {
+    return fileSystem.getXAttrs(path);
+  }
 
   /**
    * Converts a FileStatus to a CopyListingFileStatus.  If preserving ACLs,
-   * populates the CopyListingFileStatus with the ACLs.
+   * populates the CopyListingFileStatus with the ACLs. If preserving XAttrs,
+   * populates the CopyListingFileStatus with the XAttrs.
    *
    * @param fileSystem FileSystem containing the file
    * @param fileStatus FileStatus of file
    * @param preserveAcls boolean true if preserving ACLs
+   * @param preserveXAttrs boolean true if preserving XAttrs
    * @throws IOException if there is an I/O error
    */
   public static CopyListingFileStatus toCopyListingFileStatus(
-      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls)
-      throws IOException {
+      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
+      boolean preserveXAttrs) throws IOException {
     CopyListingFileStatus copyListingFileStatus =
       new CopyListingFileStatus(fileStatus);
     if (preserveAcls) {
@@ -270,6 +301,10 @@ public class DistCpUtils {
         copyListingFileStatus.setAclEntries(aclEntries);
       }
     }
+    if (preserveXAttrs) {
+      Map<String, byte[]> xAttrs = fileSystem.getXAttrs(fileStatus.getPath());
+      copyListingFileStatus.setXAttrs(xAttrs);
+    }
     return copyListingFileStatus;
   }
 
@@ -314,6 +349,25 @@ public class DistCpUtils {
         + fs.getUri());
     }
   }
+  
+  /**
+   * Determines if a file system supports XAttrs by running a getXAttrs request
+   * on the file system root. This method is used before distcp job submission
+   * to fail fast if the user requested preserving XAttrs, but the file system
+   * cannot support XAttrs.
+   * 
+   * @param fs FileSystem to check
+   * @throws XAttrsNotSupportedException if fs does not support XAttrs
+   */
+  public static void checkFileSystemXAttrSupport(FileSystem fs)
+      throws XAttrsNotSupportedException {
+    try {
+      fs.getXAttrs(new Path(Path.SEPARATOR));
+    } catch (Exception e) {
+      throw new XAttrsNotSupportedException("XAttrs not supported for file system: "
+        + fs.getUri());
+    }
+  }
 
   /**
    * String utility to convert a number-of-bytes to human readable format.

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Thu Jun 12 10:59:28 2014
@@ -414,6 +414,7 @@ public class TestOptionsParser {
     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
         "-p",
@@ -426,6 +427,7 @@ public class TestOptionsParser {
     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
         "-pbr",
@@ -439,6 +441,7 @@ public class TestOptionsParser {
     Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
         "-pbrgup",
@@ -452,9 +455,10 @@ public class TestOptionsParser {
     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
-        "-pbrgupca",
+        "-pbrgupcax",
         "-f",
         "hdfs://localhost:8020/source/first",
         "hdfs://localhost:8020/target/"});
@@ -465,6 +469,7 @@ public class TestOptionsParser {
     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertTrue(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertTrue(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
         "-pc",
@@ -478,6 +483,7 @@ public class TestOptionsParser {
     Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
     Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE));
     Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL));
+    Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR));
 
     options = OptionsParser.parse(new String[] {
         "-p",

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1602116&r1=1602115&r2=1602116&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
Thu Jun 12 10:59:28 2014
@@ -450,6 +450,7 @@ public class TestCopyMapper {
       EnumSet<DistCpOptions.FileAttribute> preserveStatus =
           EnumSet.allOf(DistCpOptions.FileAttribute.class);
       preserveStatus.remove(DistCpOptions.FileAttribute.ACL);
+      preserveStatus.remove(DistCpOptions.FileAttribute.XATTR);
 
       context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
         DistCpUtils.packAttributes(preserveStatus));
@@ -588,6 +589,7 @@ public class TestCopyMapper {
       EnumSet<DistCpOptions.FileAttribute> preserveStatus =
           EnumSet.allOf(DistCpOptions.FileAttribute.class);
       preserveStatus.remove(DistCpOptions.FileAttribute.ACL);
+      preserveStatus.remove(DistCpOptions.FileAttribute.XATTR);
 
       context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS,
         DistCpUtils.packAttributes(preserveStatus));
@@ -663,6 +665,7 @@ public class TestCopyMapper {
       EnumSet<DistCpOptions.FileAttribute> preserveStatus =
           EnumSet.allOf(DistCpOptions.FileAttribute.class);
       preserveStatus.remove(DistCpOptions.FileAttribute.ACL);
+      preserveStatus.remove(DistCpOptions.FileAttribute.XATTR);
 
       final Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
               = stubContext.getContext();



Mime
View raw message