hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1210191 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-examples/ hadoop-mapreduce-examples/dev-support/ hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/ hadoop-mapreduce-examples/s...
Date Sun, 04 Dec 2011 20:00:08 GMT
Author: mahadev
Date: Sun Dec  4 20:00:08 2011
New Revision: 1210191

URL: http://svn.apache.org/viewvc?rev=1210191&view=rev
Log:
MAPREDUCE-3458. Fix findbugs warnings in hadoop-examples. (Devaraj K via mahadev) - Merging
r1210190 from trunk.

Added:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/
      - copied from r1210190, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml
      - copied unchanged from r1210190, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml
Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Sun Dec  4 20:00:08
2011
@@ -188,6 +188,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3456. $HADOOP_PREFIX/bin/yarn should set defaults for 
     $HADOOP_*_HOME (Eric Payne via mahadev)
 
+    MAPREDUCE-3458. Fix findbugs warnings in hadoop-examples. (Devaraj K
+    via mahadev)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
Sun Dec  4 20:00:08 2011
@@ -27,6 +27,10 @@
   <name>Apache Hadoop MapReduce Examples</name>
   <packaging>jar</packaging>
 
+  <properties>
+    <mr.examples.basedir>${basedir}</mr.examples.basedir>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -59,6 +63,18 @@
        </archive>
      </configuration>
     </plugin>
+
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+         <configuration>
+          <findbugsXmlOutput>true</findbugsXmlOutput>
+          <xmlOutput>true</xmlOutput>
+          <excludeFilterFile>${mr.examples.basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
+          <effort>Max</effort>
+        </configuration>
+      </plugin>
+
    </plugins>
    </build>
 </project>

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
Sun Dec  4 20:00:08 2011
@@ -19,7 +19,9 @@
 package org.apache.hadoop.examples;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -29,9 +31,14 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.*;
+import org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat;
+import org.apache.hadoop.mapreduce.lib.join.TupleWritable;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.hadoop.util.Tool;
@@ -52,7 +59,7 @@ import org.apache.hadoop.util.ToolRunner
  *            [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i>

  */
 public class Join extends Configured implements Tool {
-  public static String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
+  public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
   static int printUsage() {
     System.out.println("join [-r <reduces>] " +
                        "[-inFormat <input format class>] " +

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
Sun Dec  4 20:00:08 2011
@@ -69,7 +69,7 @@ public class Pentomino {
     }
     
     public int[] getRotations() {
-      return rotations;
+      return rotations.clone();
     }
     
     public boolean getFlippable() {

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
Sun Dec  4 20:00:08 2011
@@ -70,7 +70,7 @@ public class TeraGen extends Configured 
 
   public static enum Counters {CHECKSUM}
 
-  public static String NUM_ROWS = "mapreduce.terasort.num-rows";
+  public static final String NUM_ROWS = "mapreduce.terasort.num-rows";
   /**
    * An input format that assigns ranges of longs to each mapper.
    */

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
Sun Dec  4 20:00:08 2011
@@ -156,10 +156,10 @@ public class TeraInputFormat extends Fil
    * them and picks N-1 keys to generate N equally sized partitions.
    * @param job the job to sample
    * @param partFile where to write the output file to
-   * @throws IOException if something goes wrong
+   * @throws Throwable if something goes wrong
    */
   public static void writePartitionFile(final JobContext job, 
-      Path partFile) throws IOException, InterruptedException  {
+      Path partFile) throws Throwable  {
     long t1 = System.currentTimeMillis();
     Configuration conf = job.getConfiguration();
     final TeraInputFormat inFormat = new TeraInputFormat();
@@ -174,11 +174,12 @@ public class TeraInputFormat extends Fil
     final long recordsPerSample = sampleSize / samples;
     final int sampleStep = splits.size() / samples;
     Thread[] samplerReader = new Thread[samples];
+    SamplerThreadGroup threadGroup = new SamplerThreadGroup("Sampler Reader Thread Group");
     // take N samples from different parts of the input
     for(int i=0; i < samples; ++i) {
       final int idx = i;
       samplerReader[i] = 
-        new Thread ("Sampler Reader " + idx) {
+        new Thread (threadGroup,"Sampler Reader " + idx) {
         {
           setDaemon(true);
         }
@@ -201,7 +202,7 @@ public class TeraInputFormat extends Fil
           } catch (IOException ie){
             System.err.println("Got an exception while reading splits " +
                 StringUtils.stringifyException(ie));
-            System.exit(-1);
+            throw new RuntimeException(ie);
           } catch (InterruptedException e) {
         	  
           }
@@ -215,6 +216,9 @@ public class TeraInputFormat extends Fil
     for (int i = 0; i < samples; i++) {
       try {
         samplerReader[i].join();
+        if(threadGroup.getThrowable() != null){
+          throw threadGroup.getThrowable();
+        }
       } catch (InterruptedException e) {
       }
     }
@@ -225,6 +229,25 @@ public class TeraInputFormat extends Fil
     long t3 = System.currentTimeMillis();
     System.out.println("Computing parititions took " + (t3 - t2) + "ms");
   }
+  
+  static class SamplerThreadGroup extends ThreadGroup{
+
+    private Throwable throwable;
+
+    public SamplerThreadGroup(String s) {
+      super(s);
+    }
+    
+    @Override
+    public void uncaughtException(Thread thread, Throwable throwable) {
+      this.throwable = throwable;
+    }
+    
+    public Throwable getThrowable() {
+      return this.throwable;
+    }
+    
+  }
 
   static class TeraRecordReader extends RecordReader<Text,Text> {
     private FSDataInputStream in;

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1210191&r1=1210190&r2=1210191&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
Sun Dec  4 20:00:08 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configured
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Cluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -300,7 +299,12 @@ public class TeraSort extends Configured
                                     TeraInputFormat.PARTITION_FILENAME);
       URI partitionUri = new URI(partitionFile.toString() +
                                  "#" + TeraInputFormat.PARTITION_FILENAME);
-      TeraInputFormat.writePartitionFile(job, partitionFile);
+      try {
+        TeraInputFormat.writePartitionFile(job, partitionFile);
+      } catch (Throwable e) {
+        LOG.error(e.getMessage());
+        return -1;
+      }
       job.addCacheFile(partitionUri);
       job.createSymlink();    
       long end = System.currentTimeMillis();



Mime
View raw message