hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1588509 - in /hadoop/common/branches/HADOOP-10388: ./ hadoop-project/ hadoop-project/src/site/ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/ hadoo...
Date Fri, 18 Apr 2014 16:32:43 GMT
Author: cmccabe
Date: Fri Apr 18 16:32:35 2014
New Revision: 1588509

URL: http://svn.apache.org/r1588509
Log:
Merge trunk into the HADOOP-10388 branch

Added:
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
      - copied unchanged from r1588387, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/
      - copied from r1588387, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/
Removed:
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Modified:
    hadoop/common/branches/HADOOP-10388/   (props changed)
    hadoop/common/branches/HADOOP-10388/BUILDING.txt
    hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml
    hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
    hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml

Propchange: hadoop/common/branches/HADOOP-10388/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk:r1582150-1588387

Modified: hadoop/common/branches/HADOOP-10388/BUILDING.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/BUILDING.txt?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/BUILDING.txt (original)
+++ hadoop/common/branches/HADOOP-10388/BUILDING.txt Fri Apr 18 16:32:35 2014
@@ -9,6 +9,8 @@ Requirements:
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
 * CMake 2.6 or newer (if compiling native code)
+* Zlib devel (if compiling native code)
+* openssl devel ( if compiling native hadoop-pipes )
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
 
 ----------------------------------------------------------------------------------
@@ -189,6 +191,7 @@ Requirements:
 * ProtocolBuffer 2.5.0
 * Windows SDK or Visual Studio 2010 Professional
 * Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
+* zlib headers (if building native code bindings for zlib)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
 
 If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
@@ -228,6 +231,18 @@ native code is built by enabling the 'na
 is enabled by default when building on Windows since the native components 
 are required (not optional) on Windows.
 
+If native code bindings for zlib are required, then the zlib headers must be
+deployed on the build machine.  Set the ZLIB_HOME environment variable to the
+directory containing the headers.
+
+set ZLIB_HOME=C:\zlib-1.2.7
+
+At runtime, zlib1.dll must be accessible on the PATH.  Hadoop has been tested
+with zlib 1.2.7, built using Visual Studio 2010 out of contrib\vstudio\vc10 in
+the zlib 1.2.7 source tree.
+
+http://www.zlib.net/
+
 ----------------------------------------------------------------------------------
 Building distributions:
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml Fri Apr 18 16:32:35 2014
@@ -631,22 +631,22 @@
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-mapper-asl</artifactId>
-        <version>1.8.8</version>
+        <version>1.9.13</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
-        <version>1.8.8</version>
+        <version>1.9.13</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-jaxrs</artifactId>
-        <version>1.8.8</version>
+        <version>1.9.13</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-xc</artifactId>
-        <version>1.8.8</version>
+        <version>1.9.13</version>
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>

Modified: hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml Fri Apr 18 16:32:35 2014
@@ -98,6 +98,7 @@
       <item name="YARN Architecture" href="hadoop-yarn/hadoop-yarn-site/YARN.html"/>
       <item name="Capacity Scheduler" href="hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html"/>
       <item name="Fair Scheduler" href="hadoop-yarn/hadoop-yarn-site/FairScheduler.html"/>
+      <item name="ResourceManager Restart" href="hadoop-yarn/hadoop-yarn-site/ResourceManagerRestart.html"/>
       <item name="Web Application Proxy" href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
       <item name="YARN Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html"/>
       <item name="Writing YARN Applications" href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java Fri Apr 18 16:32:35 2014
@@ -40,6 +40,8 @@ import org.apache.hadoop.util.ToolRunner
 import java.io.IOException;
 import java.util.Random;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * DistCp is the main driver-class for DistCpV2.
  * For command-line use, DistCp::main() orchestrates the parsing of command-line
@@ -87,7 +89,8 @@ public class DistCp extends Configured i
   /**
    * To be used with the ToolRunner. Not for public consumption.
    */
-  private DistCp() {}
+  @VisibleForTesting
+  public DistCp() {}
 
   /**
    * Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -105,7 +108,7 @@ public class DistCp extends Configured i
     
     try {
       inputOptions = (OptionsParser.parse(argv));
-
+      setTargetPathExists();
       LOG.info("Input Options: " + inputOptions);
     } catch (Throwable e) {
       LOG.error("Invalid arguments: ", e);
@@ -170,6 +173,18 @@ public class DistCp extends Configured i
   }
 
   /**
+   * Set targetPathExists in both inputOptions and job config,
+   * for the benefit of CopyCommitter
+   */
+  private void setTargetPathExists() throws IOException {
+    Path target = inputOptions.getTargetPath();
+    FileSystem targetFS = target.getFileSystem(getConf());
+    boolean targetExists = targetFS.exists(target);
+    inputOptions.setTargetPathExists(targetExists);
+    getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, 
+        targetExists);
+  }
+  /**
    * Create Job object for submitting it, with all the configuration
    *
    * @return Reference to job object.

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java Fri Apr 18 16:32:35 2014
@@ -74,6 +74,9 @@ public class DistCpConstants {
    */
   public static final String CONF_LABEL_TARGET_FINAL_PATH = "distcp.target.final.path";
 
+  /* Boolean to indicate whether the target of distcp exists. */
+  public static final String CONF_LABEL_TARGET_PATH_EXISTS = "distcp.target.path.exists";
+  
   /**
    * DistCp job id for consumers of the Disctp 
    */

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java Fri Apr 18 16:32:35 2014
@@ -60,6 +60,10 @@ public class DistCpOptions {
 
   private Path targetPath;
 
+  // targetPathExist is a derived field, it's initialized in the 
+  // beginning of distcp.
+  private boolean targetPathExists = true;
+  
   public static enum FileAttribute{
     REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE;
 
@@ -123,6 +127,7 @@ public class DistCpOptions {
       this.sourceFileListing = that.getSourceFileListing();
       this.sourcePaths = that.getSourcePaths();
       this.targetPath = that.getTargetPath();
+      this.targetPathExists = that.getTargetPathExists();
     }
   }
 
@@ -439,6 +444,22 @@ public class DistCpOptions {
     return targetPath;
   }
 
+  /**
+   * Getter for the targetPathExists.
+   * @return The target-path.
+   */
+  public boolean getTargetPathExists() {
+    return targetPathExists;
+  }
+  
+  /**
+   * Set targetPathExists.
+   * @param targetPathExists Whether the target path of distcp exists.
+   */
+  public boolean setTargetPathExists(boolean targetPathExists) {
+    return this.targetPathExists = targetPathExists;
+  }
+
   public void validate(DistCpOptionSwitch option, boolean value) {
 
     boolean syncFolder = (option == DistCpOptionSwitch.SYNC_FOLDERS ?
@@ -515,6 +536,7 @@ public class DistCpOptions {
         ", sourceFileListing=" + sourceFileListing +
         ", sourcePaths=" + sourcePaths +
         ", targetPath=" + targetPath +
+        ", targetPathExists=" + targetPathExists +
         '}';
   }
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java Fri Apr 18 16:32:35 2014
@@ -111,7 +111,24 @@ public class SimpleCopyListing extends C
   public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException {
     doBuildListing(getWriter(pathToListingFile), options);
   }
-  
+  /**
+   * Collect the list of 
+   *   <sourceRelativePath, sourceFileStatus>
+   * to be copied and write to the sequence file. In essence, any file or
+   * directory that need to be copied or sync-ed is written as an entry to the
+   * sequence file, with the possible exception of the source root:
+   *     when either -update (sync) or -overwrite switch is specified, and if
+   *     the the source root is a directory, then the source root entry is not 
+   *     written to the sequence file, because only the contents of the source
+   *     directory need to be copied in this case.
+   * See {@link org.apache.hadoop.tools.util.DistCpUtils.getRelativePath} for
+   *     how relative path is computed.
+   * See computeSourceRootPath method for how the root path of the source is
+   *     computed.
+   * @param fileListWriter
+   * @param options
+   * @throws IOException
+   */
   @VisibleForTesting
   public void doBuildListing(SequenceFile.Writer fileListWriter,
       DistCpOptions options) throws IOException {
@@ -125,7 +142,12 @@ public class SimpleCopyListing extends C
         boolean localFile = (rootStatus.getClass() != FileStatus.class);
 
         FileStatus[] sourceFiles = sourceFS.listStatus(path);
-        if (sourceFiles != null && sourceFiles.length > 0) {
+        boolean explore = (sourceFiles != null && sourceFiles.length > 0);
+        if (!explore || rootStatus.isDirectory()) {
+          writeToFileListingRoot(fileListWriter, rootStatus, sourcePathRoot,
+              localFile, options);
+        }
+        if (explore) {
           for (FileStatus sourceStatus: sourceFiles) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
@@ -141,9 +163,6 @@ public class SimpleCopyListing extends C
                   localFile, options);
             }
           }
-        } else {
-          writeToFileListing(fileListWriter, rootStatus, sourcePathRoot,
-              localFile, options);
         }
       }
       fileListWriter.close();
@@ -158,18 +177,19 @@ public class SimpleCopyListing extends C
 
     Path target = options.getTargetPath();
     FileSystem targetFS = target.getFileSystem(getConf());
+    final boolean targetPathExists = options.getTargetPathExists();
 
     boolean solitaryFile = options.getSourcePaths().size() == 1
                                                 && !sourceStatus.isDirectory();
 
     if (solitaryFile) {
-      if (targetFS.isFile(target) || !targetFS.exists(target)) {
+      if (targetFS.isFile(target) || !targetPathExists) {
         return sourceStatus.getPath();
       } else {
         return sourceStatus.getPath().getParent();
       }
     } else {
-      boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetFS.exists(target)) ||
+      boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetPathExists) ||
           options.shouldSyncFolder() || options.shouldOverwrite();
 
       return specialHandling && sourceStatus.isDirectory() ? sourceStatus.getPath() :
@@ -253,15 +273,30 @@ public class SimpleCopyListing extends C
       }
     }
   }
+  
+  private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
+      FileStatus fileStatus, Path sourcePathRoot,
+      boolean localFile,
+      DistCpOptions options) throws IOException {
+    boolean syncOrOverwrite = options.shouldSyncFolder() ||
+        options.shouldOverwrite();
+    if (fileStatus.getPath().equals(sourcePathRoot) && 
+        fileStatus.isDirectory() && syncOrOverwrite) {
+      // Skip the root-paths when syncOrOverwrite
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skip " + fileStatus.getPath());
+      }      
+      return;
+    }
+    writeToFileListing(fileListWriter, fileStatus, sourcePathRoot, localFile,
+        options);
+  }
 
   private void writeToFileListing(SequenceFile.Writer fileListWriter,
                                   FileStatus fileStatus,
                                   Path sourcePathRoot,
                                   boolean localFile,
                                   DistCpOptions options) throws IOException {
-    if (fileStatus.getPath().equals(sourcePathRoot) && fileStatus.isDirectory())
-      return; // Skip the root-paths.
-
     if (LOG.isDebugEnabled()) {
       LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot,
         fileStatus.getPath()) + ", FULL PATH: " + fileStatus.getPath());

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java Fri Apr 18 16:32:35 2014
@@ -54,7 +54,10 @@ public class CopyCommitter extends FileO
   private static final Log LOG = LogFactory.getLog(CopyCommitter.class);
 
   private final TaskAttemptContext taskAttemptContext;
-
+  private boolean syncFolder = false;
+  private boolean overwrite = false;
+  private boolean targetPathExists = true;
+  
   /**
    * Create a output committer
    *
@@ -71,6 +74,10 @@ public class CopyCommitter extends FileO
   @Override
   public void commitJob(JobContext jobContext) throws IOException {
     Configuration conf = jobContext.getConfiguration();
+    syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
+    overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
+    targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
+    
     super.commitJob(jobContext);
 
     cleanupTempFiles(jobContext);
@@ -155,6 +162,8 @@ public class CopyCommitter extends FileO
   // user/group permissions, etc.) based on the corresponding source directories.
   private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
     String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
+    final boolean syncOrOverwrite = syncFolder || overwrite;
+
     LOG.info("About to preserve attributes: " + attrSymbols);
 
     EnumSet<FileAttribute> attributes = DistCpUtils.unpackAttributes(attrSymbols);
@@ -179,12 +188,10 @@ public class CopyCommitter extends FileO
         if (! srcFileStatus.isDirectory()) continue;
 
         Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
-
-        // Skip the root folder.
-        // Status can't be preserved on root-folder. (E.g. multiple paths may
-        // be copied to a single target folder. Which source-attributes to use
-        // on the target is undefined.)
-        if (targetRoot.equals(targetFile)) continue;
+        //
+        // Skip the root folder when syncOrOverwrite is true.
+        //
+        if (targetRoot.equals(targetFile) && syncOrOverwrite) continue;
 
         FileSystem targetFS = targetFile.getFileSystem(conf);
         DistCpUtils.preserve(targetFS, targetFile, srcFileStatus,  attributes);
@@ -218,7 +225,14 @@ public class CopyCommitter extends FileO
     Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
     targets.add(targetFinalPath);
     DistCpOptions options = new DistCpOptions(targets, new Path("/NONE"));
-
+    //
+    // Set up options to be the same from the CopyListing.buildListing's perspective,
+    // so to collect similar listings as when doing the copy
+    //
+    options.setOverwrite(overwrite);
+    options.setSyncFolder(syncFolder);
+    options.setTargetPathExists(targetPathExists);
+    
     target.buildListing(targetListing, options);
     Path sortedTargetListing = DistCpUtils.sortListing(clusterFS, conf, targetListing);
     long totalLen = clusterFS.getFileStatus(sortedTargetListing).getLen();

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java Fri Apr 18 16:32:35 2014
@@ -103,12 +103,14 @@ public class TestCopyListing extends Sim
     DistCpOptions options = new DistCpOptions(srcPaths, target);
     Path listingFile = new Path("/tmp/list4");
     listing.buildListing(listingFile, options);
-    Assert.assertEquals(listing.getNumberOfPaths(), 2);
+    Assert.assertEquals(listing.getNumberOfPaths(), 3);
     SequenceFile.Reader reader = new SequenceFile.Reader(getConf(),
         SequenceFile.Reader.file(listingFile));
     FileStatus fileStatus = new FileStatus();
     Text relativePath = new Text();
     Assert.assertTrue(reader.next(relativePath, fileStatus));
+    Assert.assertEquals(relativePath.toString(), "/1");
+    Assert.assertTrue(reader.next(relativePath, fileStatus));
     Assert.assertEquals(relativePath.toString(), "/1/file");
     Assert.assertTrue(reader.next(relativePath, fileStatus));
     Assert.assertEquals(relativePath.toString(), "/2");
@@ -270,7 +272,8 @@ public class TestCopyListing extends Sim
       final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
       listing.buildListing(listFile, options);
 
-      reader = new SequenceFile.Reader(fs, listFile, getConf());
+      reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile));
+
       FileStatus fileStatus = new FileStatus();
       Text relativePath = new Text();
       Assert.assertTrue(reader.next(relativePath, fileStatus));

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java Fri Apr 18 16:32:35 2014
@@ -85,7 +85,7 @@ public class TestDistCpViewFs {
       addEntries(listFile, "singlefile1/file1");
       createFiles("singlefile1/file1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1);
     } catch (IOException e) {
@@ -108,7 +108,7 @@ public class TestDistCpViewFs {
       addEntries(listFile, "singlefile1/file1");
       createFiles("singlefile1/file1", target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1);
     } catch (IOException e) {
@@ -132,7 +132,7 @@ public class TestDistCpViewFs {
       createFiles("singlefile2/file2");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(target, 1, "file2");
     } catch (IOException e) {
@@ -155,7 +155,7 @@ public class TestDistCpViewFs {
       addEntries(listFile, "singledir");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1, "dir1");
     } catch (IOException e) {
@@ -174,7 +174,7 @@ public class TestDistCpViewFs {
       mkdirs(root + "/singledir/dir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, true, false);
 
       checkResult(target, 1, "singledir/dir1");
     } catch (IOException e) {
@@ -193,7 +193,7 @@ public class TestDistCpViewFs {
       mkdirs(root + "/Usingledir/Udir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, true, true);
 
       checkResult(target, 1, "Udir1");
     } catch (IOException e) {
@@ -217,7 +217,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(target, 3, "file3", "file4", "file5");
     } catch (IOException e) {
@@ -240,7 +240,7 @@ public class TestDistCpViewFs {
       addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 3, "file3", "file4", "file5");
     } catch (IOException e) {
@@ -259,7 +259,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(target.toString(), root + "/singledir/dir1");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, true, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5", "singledir/dir1");
     } catch (IOException e) {
@@ -278,7 +278,7 @@ public class TestDistCpViewFs {
       createFiles("Umultifile/Ufile3", "Umultifile/Ufile4", "Umultifile/Ufile5");
       mkdirs(target.toString(), root + "/Usingledir/Udir1");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, true, true);
 
       checkResult(target, 4, "Ufile3", "Ufile4", "Ufile5", "Udir1");
     } catch (IOException e) {
@@ -297,7 +297,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4",
           "multifile/file5", "singledir/dir1");
@@ -317,7 +317,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 4, "file3", "file4", "file5", "dir1");
     } catch (IOException e) {
@@ -338,7 +338,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("singledir/dir2/file6");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5",
           "singledir/dir2/file6");
@@ -361,7 +361,7 @@ public class TestDistCpViewFs {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("singledir/dir2/file6");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 4, "file3", "file4", "file5", "dir2/file6");
     } catch (IOException e) {
@@ -384,7 +384,7 @@ public class TestDistCpViewFs {
       createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
           "singledir1/dir3/file9");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 4, "file3", "file4", "file5",
           "dir3/file7", "dir3/file8", "dir3/file9");
@@ -408,7 +408,7 @@ public class TestDistCpViewFs {
       createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
           "singledir1/dir3/file9");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 6, "file3", "file4", "file5",
           "file7", "file8", "file9");
@@ -460,9 +460,11 @@ public class TestDistCpViewFs {
     }
   }
 
-  private void runTest(Path listFile, Path target, boolean sync) throws IOException {
+  private void runTest(Path listFile, Path target, boolean targetExists, 
+      boolean sync) throws IOException {
     DistCpOptions options = new DistCpOptions(listFile, target);
     options.setSyncFolder(sync);
+    options.setTargetPathExists(targetExists);
     try {
       new DistCp(getConf(), options).execute();
     } catch (Exception e) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java Fri Apr 18 16:32:35 2014
@@ -112,7 +112,7 @@ public class TestFileBasedCopyListing {
       addEntries(listFile, "/tmp/singlefile1/file1");
       createFiles("/tmp/singlefile1/file1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(listFile, 0);
     } catch (IOException e) {
@@ -138,7 +138,7 @@ public class TestFileBasedCopyListing {
       addEntries(listFile, "/tmp/singlefile1/file1");
       createFiles("/tmp/singlefile1/file1", target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(listFile, 0);
     } catch (IOException e) {
@@ -165,7 +165,7 @@ public class TestFileBasedCopyListing {
       createFiles("/tmp/singlefile2/file2");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(listFile, 1);
     } catch (IOException e) {
@@ -191,7 +191,7 @@ public class TestFileBasedCopyListing {
       addEntries(listFile, "/tmp/singledir");
       mkdirs("/tmp/singledir/dir1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(listFile, 1);
     } catch (IOException e) {
@@ -213,7 +213,7 @@ public class TestFileBasedCopyListing {
       mkdirs("/tmp/singledir/dir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target);
+      runTest(listFile, target, true);
 
       checkResult(listFile, 1);
     } catch (IOException e) {
@@ -235,7 +235,7 @@ public class TestFileBasedCopyListing {
       mkdirs("/tmp/Usingledir/Udir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, true, true);
 
       checkResult(listFile, 1);
     } catch (IOException e) {
@@ -262,7 +262,7 @@ public class TestFileBasedCopyListing {
       createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(listFile, 3);
     } catch (IOException e) {
@@ -288,7 +288,7 @@ public class TestFileBasedCopyListing {
       addEntries(listFile, "/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
       createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(listFile, 3);
     } catch (IOException e) {
@@ -310,7 +310,7 @@ public class TestFileBasedCopyListing {
       createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
       mkdirs(target.toString(), "/tmp/singledir/dir1");
 
-      runTest(listFile, target);
+      runTest(listFile, target, true);
 
       checkResult(listFile, 4);
     } catch (IOException e) {
@@ -440,7 +440,7 @@ public class TestFileBasedCopyListing {
           "/tmp/singledir1/dir3/file9");
       mkdirs(target.toString());
 
-      runTest(listFile, target);
+      runTest(listFile, target, true);
 
       checkResult(listFile, 6);
     } catch (IOException e) {
@@ -507,14 +507,17 @@ public class TestFileBasedCopyListing {
     }
   }
 
-  private void runTest(Path listFile, Path target) throws IOException {
-    runTest(listFile, target, true);
+  private void runTest(Path listFile, Path target,
+      boolean targetExists) throws IOException {
+    runTest(listFile, target, targetExists, true);
   }
 
-  private void runTest(Path listFile, Path target, boolean sync) throws IOException {
+  private void runTest(Path listFile, Path target, boolean targetExists,
+      boolean sync) throws IOException {
     CopyListing listing = new FileBasedCopyListing(config, CREDENTIALS);
     DistCpOptions options = new DistCpOptions(listFile, target);
     options.setSyncFolder(sync);
+    options.setTargetPathExists(targetExists);
     listing.buildListing(listFile, options);
   }
 
@@ -530,6 +533,11 @@ public class TestFileBasedCopyListing {
       Text relPath = new Text();
       FileStatus fileStatus = new FileStatus();
       while (reader.next(relPath, fileStatus)) {
+        if (fileStatus.isDirectory() && relPath.toString().equals("")) {
+          // ignore root with empty relPath, which is an entry to be 
+          // used for preserving root attributes etc.
+          continue;
+        }
         Assert.assertEquals(fileStatus.getPath().toUri().getPath(), map.get(relPath.toString()));
         recCount++;
       }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java Fri Apr 18 16:32:35 2014
@@ -111,7 +111,7 @@ public class TestGlobbedCopyListing {
     Path target = new Path(fileSystemPath.toString() + "/tmp/target");
     Path listingPath = new Path(fileSystemPath.toString() + "/tmp/META/fileList.seq");
     DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);
-
+    options.setTargetPathExists(false);
     new GlobbedCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options);
 
     verifyContents(listingPath);
@@ -124,6 +124,11 @@ public class TestGlobbedCopyListing {
     FileStatus value = new FileStatus();
     Map<String, String> actualValues = new HashMap<String, String>();
     while (reader.next(key, value)) {
+      if (value.isDirectory() && key.toString().equals("")) {
+        // ignore root with empty relPath, which is an entry to be 
+        // used for preserving root attributes etc.
+        continue;
+      }
       actualValues.put(value.getPath().toString(), key.toString());
     }
 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java Fri Apr 18 16:32:35 2014
@@ -82,7 +82,7 @@ public class TestIntegration {
       addEntries(listFile, "singlefile1/file1");
       createFiles("singlefile1/file1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1);
     } catch (IOException e) {
@@ -105,7 +105,7 @@ public class TestIntegration {
       addEntries(listFile, "singlefile1/file1");
       createFiles("singlefile1/file1", "target");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1);
     } catch (IOException e) {
@@ -129,7 +129,7 @@ public class TestIntegration {
       createFiles("singlefile2/file2");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(target, 1, "file2");
     } catch (IOException e) {
@@ -152,7 +152,7 @@ public class TestIntegration {
       addEntries(listFile, "singledir");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 1, "dir1");
     } catch (IOException e) {
@@ -171,7 +171,7 @@ public class TestIntegration {
       mkdirs(root + "/singledir/dir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, true, false);
 
       checkResult(target, 1, "singledir/dir1");
     } catch (IOException e) {
@@ -190,7 +190,7 @@ public class TestIntegration {
       mkdirs(root + "/Usingledir/Udir1");
       mkdirs(target.toString());
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, true, true);
 
       checkResult(target, 1, "Udir1");
     } catch (IOException e) {
@@ -214,7 +214,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(target.toString());
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, true, sync);
 
       checkResult(target, 3, "file3", "file4", "file5");
     } catch (IOException e) {
@@ -286,7 +286,7 @@ public class TestIntegration {
       addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
 
-      runTest(listFile, target, sync);
+      runTest(listFile, target, false, sync);
 
       checkResult(target, 3, "file3", "file4", "file5");
     } catch (IOException e) {
@@ -305,7 +305,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(target.toString(), root + "/singledir/dir1");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, true, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5", "singledir/dir1");
     } catch (IOException e) {
@@ -324,7 +324,7 @@ public class TestIntegration {
       createFiles("Umultifile/Ufile3", "Umultifile/Ufile4", "Umultifile/Ufile5");
       mkdirs(target.toString(), root + "/Usingledir/Udir1");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, true, true);
 
       checkResult(target, 4, "Ufile3", "Ufile4", "Ufile5", "Udir1");
     } catch (IOException e) {
@@ -343,7 +343,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4",
           "multifile/file5", "singledir/dir1");
@@ -363,7 +363,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       mkdirs(root + "/singledir/dir1");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 4, "file3", "file4", "file5", "dir1");
     } catch (IOException e) {
@@ -382,7 +382,7 @@ public class TestIntegration {
       createFiles("srcdir/file1", "dstdir/file1", "dstdir/file2");
       
       Path target = new Path(root + "/dstdir");
-      runTest(listFile, target, true, true, false);
+      runTest(listFile, target, false, true, true, false);
       
       checkResult(target, 1, "file1");
     } catch (IOException e) {
@@ -406,7 +406,7 @@ public class TestIntegration {
       createWithContents("dstdir/file1", contents2);
       
       Path target = new Path(root + "/dstdir");
-      runTest(listFile, target, false, false, true);
+      runTest(listFile, target, false, false, false, true);
       
       checkResult(target, 1, "file1");
       
@@ -436,7 +436,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("singledir/dir2/file6");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5",
           "singledir/dir2/file6");
@@ -459,7 +459,7 @@ public class TestIntegration {
       createFiles("multifile/file3", "multifile/file4", "multifile/file5");
       createFiles("singledir/dir2/file6");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 4, "file3", "file4", "file5", "dir2/file6");
     } catch (IOException e) {
@@ -482,7 +482,7 @@ public class TestIntegration {
       createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
           "singledir1/dir3/file9");
 
-      runTest(listFile, target, false);
+      runTest(listFile, target, false, false);
 
       checkResult(target, 4, "file3", "file4", "file5",
           "dir3/file7", "dir3/file8", "dir3/file9");
@@ -506,7 +506,7 @@ public class TestIntegration {
       createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
           "singledir1/dir3/file9");
 
-      runTest(listFile, target, true);
+      runTest(listFile, target, false, true);
 
       checkResult(target, 6, "file3", "file4", "file5",
           "file7", "file8", "file9");
@@ -584,16 +584,19 @@ public class TestIntegration {
     }
   }
     
-  private void runTest(Path listFile, Path target, boolean sync) throws IOException {
-    runTest(listFile, target, sync, false, false);
+  private void runTest(Path listFile, Path target, boolean targetExists,
+      boolean sync) throws IOException {
+    runTest(listFile, target, targetExists, sync, false, false);
   }
   
-  private void runTest(Path listFile, Path target, boolean sync, boolean delete,
+  private void runTest(Path listFile, Path target, boolean targetExists, 
+      boolean sync, boolean delete,
       boolean overwrite) throws IOException {
     DistCpOptions options = new DistCpOptions(listFile, target);
     options.setSyncFolder(sync);
     options.setDeleteMissing(delete);
     options.setOverwrite(overwrite);
+    options.setTargetPathExists(targetExists);
     try {
       new DistCp(getConf(), options).execute();
     } catch (Exception e) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java Fri Apr 18 16:32:35 2014
@@ -354,7 +354,7 @@ public class TestOptionsParser {
     DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz"));
     String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " +
         "ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " +
-        "sourceFileListing=abc, sourcePaths=null, targetPath=xyz}";
+        "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, targetPathExists=true}";
     Assert.assertEquals(val, option.toString());
     Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
         DistCpOptionSwitch.ATOMIC_COMMIT.name());

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java Fri Apr 18 16:32:35 2014
@@ -146,7 +146,8 @@ public class TestCopyCommitter {
           new Path("/out"));
       options.preserve(FileAttribute.PERMISSION);
       options.appendToConf(conf);
-
+      options.setTargetPathExists(false);
+      
       CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
       Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
       listing.buildListing(listingFile, options);

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java Fri Apr 18 16:32:35 2014
@@ -313,7 +313,7 @@ public class TestCopyMapper {
               = stubContext.getContext();
 
       Configuration configuration = context.getConfiguration();
-      String workPath = new Path("hftp://localhost:1234/*/*/*/?/")
+      String workPath = new Path("webhdfs://localhost:1234/*/*/*/?/")
               .makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
       configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
               workPath);

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java Fri Apr 18 16:32:35 2014
@@ -133,7 +133,9 @@ public class TestUniformSizeInputFormat 
         Path sourcePath = recordReader.getCurrentValue().getPath();
         FileSystem fs = sourcePath.getFileSystem(configuration);
         FileStatus fileStatus [] = fs.listStatus(sourcePath);
-        Assert.assertEquals(fileStatus.length, 1);
+        if (fileStatus.length > 1) {
+          continue;
+        }
         currentSplitSize += fileStatus[0].getLen();
       }
       Assert.assertTrue(

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java Fri Apr 18 16:32:35 2014
@@ -59,7 +59,9 @@ public class TestDynamicInputFormat {
 
     for (int i=0; i<N_FILES; ++i)
       createFile("/tmp/source/" + String.valueOf(i));
-
+    FileSystem fileSystem = cluster.getFileSystem();
+    expectedFilePaths.add(fileSystem.listStatus(
+        new Path("/tmp/source/0"))[0].getPath().getParent().toString());
   }
 
   private static Configuration getConfigurationForCluster() {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java Fri Apr 18 16:32:35 2014
@@ -891,64 +891,6 @@ public class TestCopyFiles extends TestC
     return home;
   }
 
-  public void testHftpAccessControl() throws Exception {
-    MiniDFSCluster cluster = null;
-    try {
-      final UserGroupInformation DFS_UGI = createUGI("dfs", true); 
-      final UserGroupInformation USER_UGI = createUGI("user", false); 
-
-      //start cluster by DFS_UGI
-      final Configuration dfsConf = new Configuration();
-      cluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(2).build();
-      cluster.waitActive();
-
-      final String httpAdd = dfsConf.get("dfs.http.address");
-      final URI nnURI = FileSystem.getDefaultUri(dfsConf);
-      final String nnUri = nnURI.toString();
-      FileSystem fs1 = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
-        public FileSystem run() throws IOException {
-          return FileSystem.get(nnURI, dfsConf);
-        }
-      });
-      final Path home = 
-        createHomeDirectory(fs1, USER_UGI);
-      
-      //now, login as USER_UGI
-      final Configuration userConf = new Configuration();
-      final FileSystem fs = 
-        USER_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
-        public FileSystem run() throws IOException {
-          return FileSystem.get(nnURI, userConf);
-        }
-      });
-      
-      final Path srcrootpath = new Path(home, "src_root"); 
-      final String srcrootdir =  srcrootpath.toString();
-      final Path dstrootpath = new Path(home, "dst_root"); 
-      final String dstrootdir =  dstrootpath.toString();
-      final DistCpV1 distcp = USER_UGI.doAs(new PrivilegedExceptionAction<DistCpV1>() {
-        public DistCpV1 run() {
-          return new DistCpV1(userConf);
-        }
-      });
-
-      FileSystem.mkdirs(fs, srcrootpath, new FsPermission((short)0700));
-      final String[] args = {"hftp://"+httpAdd+srcrootdir, nnUri+dstrootdir};
-
-      { //copy with permission 000, should fail
-        fs.setPermission(srcrootpath, new FsPermission((short)0));
-        USER_UGI.doAs(new PrivilegedExceptionAction<Void>() {
-          public Void run() throws Exception {
-            assertEquals(-3, ToolRunner.run(distcp, args));
-            return null;
-          }
-        });
-      }
-    } finally {
-      if (cluster != null) { cluster.shutdown(); }
-    }
-  }
-
   /** test -delete */
   public void testDelete() throws Exception {
     final Configuration conf = new Configuration();

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java Fri Apr 18 16:32:35 2014
@@ -24,7 +24,7 @@ import org.codehaus.jackson.JsonParser;
 import org.codehaus.jackson.JsonProcessingException;
 import org.codehaus.jackson.map.DeserializationContext;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.deser.StdDeserializer;
+import org.codehaus.jackson.map.deser.std.StdDeserializer;
 import org.codehaus.jackson.node.ObjectNode;
 
 /**

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java Fri Apr 18 16:32:35 2014
@@ -120,7 +120,7 @@ public class RumenToSLSConverter {
       Writer output = new FileWriter(outputFile);
       try {
         ObjectMapper mapper = new ObjectMapper();
-        ObjectWriter writer = mapper.defaultPrettyPrintingWriter();
+        ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
         Iterator<Map> i = mapper.readValues(
                 new JsonFactory().createJsonParser(input), Map.class);
         while (i.hasNext()) {
@@ -141,7 +141,7 @@ public class RumenToSLSConverter {
     Writer output = new FileWriter(outputFile);
     try {
       ObjectMapper mapper = new ObjectMapper();
-      ObjectWriter writer = mapper.defaultPrettyPrintingWriter();
+      ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
       for (Map.Entry<String, Set<String>> entry : rackNodeMap.entrySet()) {
         Map rack = new LinkedHashMap();
         rack.put("rack", entry.getKey());

Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml Fri Apr 18 16:32:35 2014
@@ -16,6 +16,18 @@
    limitations under the License.
 -->
 <FindBugsFilter>
+  <!-- Workaround bugs in findbugs 1.3.9. See http://sourceforge.net/p/findbugs/bugs/918 for more details. -->
+  <Match>
+    <Or>
+      <Package name="org.apache.hadoop.streaming" />
+    </Or>
+    <Bug pattern="NP_ALWAYS_NULL"/>
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.streaming.JarBuilder" />
+    <Bug pattern="NP_NULL_PARAM_DEREF_ALL_TARGETS_DANGEROUS"/>
+  </Match>
+
   <Match>
     <Or>
       <Class name="org.apache.hadoop.streaming.PipeMapper" />
@@ -27,4 +39,13 @@
     </Or>
     <Bug pattern="EI_EXPOSE_REP"/>
   </Match>
+
+  <Match>
+    <Package name="org.apache.hadoop.record" />
+    <Or>
+      <Bug pattern="EI_EXPOSE_REP" />
+      <Bug pattern="EI_EXPOSE_REP2" />
+      <Bug pattern="MS_PKGPROTECT" />
+    </Or>
+  </Match>
 </FindBugsFilter>



Mime
View raw message