hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1383030 - in /hadoop/common/branches/HDFS-3077: ./ hadoop-client/ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/ hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/ hadoop-tools/hadoop-distcp/src/tes...
Date Mon, 10 Sep 2012 18:45:50 GMT
Author: todd
Date: Mon Sep 10 18:45:45 2012
New Revision: 1383030

URL: http://svn.apache.org/viewvc?rev=1383030&view=rev
Log:
Merge trunk into QJM branch

Modified:
    hadoop/common/branches/HDFS-3077/   (props changed)
    hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml
    hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
    hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
    hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
    hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
    hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java

Propchange: hadoop/common/branches/HDFS-3077/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk:r1380987-1383029

Modified: hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml Mon Sep 10 18:45:45 2012
@@ -40,10 +40,6 @@
       <scope>compile</scope>
       <exclusions>
         <exclusion>
-          <groupId>commons-cli</groupId>
-          <artifactId>commons-cli</artifactId>
-        </exclusion>
-        <exclusion>
           <groupId>commons-httpclient</groupId>
           <artifactId>commons-httpclient</artifactId>
         </exclusion>

Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
Mon Sep 10 18:45:45 2012
@@ -98,6 +98,11 @@ public class DistCp extends Configured i
    * @return On success, it returns 0. Else, -1.
    */
   public int run(String[] argv) {
+    if (argv.length < 1) {
+      OptionsParser.usage();
+      return DistCpConstants.INVALID_ARGUMENT;
+    }
+    
     try {
       inputOptions = (OptionsParser.parse(argv));
 

Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
Mon Sep 10 18:45:45 2012
@@ -156,6 +156,10 @@ public class OptionsParser {
       try {
         Integer mapBandwidth = Integer.parseInt(
             getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
+        if (mapBandwidth.intValue() <= 0) {
+          throw new IllegalArgumentException("Bandwidth specified is not positive: " +
+              mapBandwidth);
+        }
         option.setMapBandwidth(mapBandwidth);
       } catch (NumberFormatException e) {
         throw new IllegalArgumentException("Bandwidth specified is invalid: " +

Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
Mon Sep 10 18:45:45 2012
@@ -255,7 +255,7 @@ public class CopyMapper extends Mapper<T
 
     long bytesCopied;
     try {
-      bytesCopied = (Long)new RetriableFileCopyCommand(description)
+      bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description)
                        .execute(sourceFileStatus, target, context, fileAttributes);
     } catch (Exception e) {
       context.setStatus("Copy Failure: " + sourceFileStatus.getPath());

Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
Mon Sep 10 18:45:45 2012
@@ -41,7 +41,8 @@ public class RetriableFileCopyCommand ex
 
   private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class);
   private static int BUFFER_SIZE = 8 * 1024;
-
+  private boolean skipCrc = false;
+  
   /**
    * Constructor, taking a description of the action.
    * @param description Verbose description of the copy operation.
@@ -49,6 +50,17 @@ public class RetriableFileCopyCommand ex
   public RetriableFileCopyCommand(String description) {
     super(description);
   }
+ 
+  /**
+   * Create a RetriableFileCopyCommand.
+   *
+   * @param skipCrc Whether to skip the crc check.
+   * @param description A verbose description of the copy operation.
+   */
+  public RetriableFileCopyCommand(boolean skipCrc, String description) {
+    this(description);
+    this.skipCrc = skipCrc;
+  }
 
   /**
    * Implementation of RetriableCommand::doExecute().
@@ -92,7 +104,7 @@ public class RetriableFileCopyCommand ex
 
       compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead);
       //At this point, src&dest lengths are same. if length==0, we skip checksum
-      if (bytesRead != 0) { 
+      if ((bytesRead != 0) && (!skipCrc)) {
         compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath);
       }
       promoteTmpToTarget(tmpTargetPath, target, targetFS);

Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Mon Sep 10 18:45:45 2012
@@ -110,6 +110,24 @@ public class TestOptionsParser {
         "hdfs://localhost:8020/target/"});
     Assert.assertEquals(options.getMapBandwidth(), 11);
   }
+  
+  @Test(expected=IllegalArgumentException.class)
+  public void testParseNonPositiveBandwidth() {
+    OptionsParser.parse(new String[] {
+        "-bandwidth",
+        "-11",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+  }
+  
+  @Test(expected=IllegalArgumentException.class)
+  public void testParseZeroBandwidth() {
+    OptionsParser.parse(new String[] {
+        "-bandwidth",
+        "0",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+  }
 
   @Test
   public void testParseSkipCRC() {



Mime
View raw message