hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r773655 - in /hadoop/core/trunk: CHANGES.txt src/tools/org/apache/hadoop/tools/DistCp.java
Date Mon, 11 May 2009 18:50:12 GMT
Author: cdouglas
Date: Mon May 11 18:50:11 2009
New Revision: 773655

URL: http://svn.apache.org/viewvc?rev=773655&view=rev
Log:
HADOOP-5675. Do not launch a job if DistCp has no work to do. Contributed by Tsz Wo (Nicholas),
SZE

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=773655&r1=773654&r2=773655&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 11 18:50:11 2009
@@ -549,6 +549,9 @@
     HADOOP-5476. Close the underlying InputStream in SequenceFile::Reader when
     the constructor throws an exception. (Michael Tamm via cdouglas)
 
+    HADOOP-5675. Do not launch a job if DistCp has no work to do. (Tsz Wo
+    (Nicholas), SZE via cdouglas)
+
 Release 0.20.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=773655&r1=773654&r2=773655&view=diff
==============================================================================
--- hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Mon May 11 18:50:11 2009
@@ -647,8 +647,9 @@
     
     //Initialize the mapper
     try {
-      setup(conf, job, args);
-      JobClient.runJob(job);
+      if (setup(conf, job, args)) {
+        JobClient.runJob(job);
+      }
       finalize(conf, job, args.dst, args.preservedAttributes);
     } finally {
       //delete tmp
@@ -968,8 +969,9 @@
    * @param conf : The dfs/mapred configuration.
    * @param jobConf : The handle to the jobConf object to be initialized.
    * @param args Arguments
+   * @return true if it is necessary to launch a job.
    */
-  private static void setup(Configuration conf, JobConf jobConf,
+  private static boolean setup(Configuration conf, JobConf jobConf,
                             final Arguments args)
       throws IOException {
     jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
@@ -1139,10 +1141,13 @@
         (dstExists && !dstIsDir) || (!dstExists && srcCount == 1)?
         args.dst.getParent(): args.dst, "_distcp_tmp_" + randomId);
     jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
-    LOG.info("srcCount=" + srcCount);
+    LOG.info("sourcePathsCount=" + srcCount);
+    LOG.info("filesToCopyCount=" + fileCount);
+    LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
     jobConf.setInt(SRC_COUNT_LABEL, srcCount);
     jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
     setMapCount(byteCount, jobConf);
+    return fileCount > 0;
   }
 
   /**



Mime
View raw message