hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r768435 - in /hadoop/hbase/branches/0.19: CHANGES.txt src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
Date Fri, 24 Apr 2009 22:45:17 GMT
Author: stack
Date: Fri Apr 24 22:45:16 2009
New Revision: 768435

URL: http://svn.apache.org/viewvc?rev=768435&view=rev
Log:
HBASE-1287 Partitioner class not used in TableMapReduceUtil.initTableReduceJob()

Modified:
    hadoop/hbase/branches/0.19/CHANGES.txt
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java

Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=768435&r1=768434&r2=768435&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Fri Apr 24 22:45:16 2009
@@ -10,6 +10,8 @@
                (Rong-en Fan via Stack)
    HBASE-1292  php thrift's getRow() would throw an exception if the row does
                not exist (Rong-en Fan via Stack)
+   HBASE-1287  Partitioner class not used in TableMapReduceUtil.initTableReduceJob()
+               (Lars George and Billy Pearson via Stack)
 
 Release 0.19.1 - 03/19/2009
   BUG FIXES

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=768435&r1=768434&r2=768435&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
(original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
Fri Apr 24 22:45:16 2009
@@ -35,16 +35,17 @@
  */
 @SuppressWarnings("unchecked")
 public class TableMapReduceUtil {
+  
   /**
    * Use this before submitting a TableMap job. It will
    * appropriately set up the JobConf.
    * 
-   * @param table table name
-   * @param columns columns to scan
-   * @param mapper mapper class
-   * @param outputKeyClass
-   * @param outputValueClass
-   * @param job job configuration
+   * @param table  The table name to read from.
+   * @param columns  The columns to scan.
+   * @param mapper  The mapper class to use.
+   * @param outputKeyClass  The class of the output key.
+   * @param outputValueClass  The class of the output value.
+   * @param job  The current job configuration to adjust.
    */
   public static void initTableMapJob(String table, String columns,
     Class<? extends TableMap> mapper, 
@@ -63,10 +64,10 @@
    * Use this before submitting a TableReduce job. It will
    * appropriately set up the JobConf.
    * 
-   * @param table
-   * @param reducer
-   * @param job
-   * @throws IOException 
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When determining the region count fails. 
    */
   public static void initTableReduceJob(String table,
     Class<? extends TableReduce> reducer, JobConf job)
@@ -78,12 +79,12 @@
    * Use this before submitting a TableReduce job. It will
    * appropriately set up the JobConf.
    * 
-   * @param table
-   * @param reducer
-   * @param job
-   * @param partitioner Partitioner to use. Pass null to use default
-   * partitioner.
-   * @throws IOException 
+   * @param table  The output table.
+   * @param reducer  The reducer class to use.
+   * @param job  The current job configuration to adjust.
+   * @param partitioner  Partitioner to use. Pass <code>null</code> to use 
+   * default partitioner.
+   * @throws IOException When determining the region count fails. 
    */
   public static void initTableReduceJob(String table,
     Class<? extends TableReduce> reducer, JobConf job, Class partitioner)
@@ -93,13 +94,78 @@
     job.set(TableOutputFormat.OUTPUT_TABLE, table);
     job.setOutputKeyClass(ImmutableBytesWritable.class);
     job.setOutputValueClass(BatchUpdate.class);
-    if (partitioner != null) {
+    if (partitioner == HRegionPartitioner.class) {
       job.setPartitionerClass(HRegionPartitioner.class);
       HTable outputTable = new HTable(new HBaseConfiguration(job), table);
       int regions = outputTable.getRegionsInfo().size();
-      if (job.getNumReduceTasks() > regions){
-    	job.setNumReduceTasks(outputTable.getRegionsInfo().size());
+      if (job.getNumReduceTasks() > regions) {
+        job.setNumReduceTasks(outputTable.getRegionsInfo().size());
       }
+    } else if (partitioner != null) {
+      job.setPartitionerClass(partitioner);
     }
   }
+  
+  /**
+   * Ensures that the given number of reduce tasks for the given job 
+   * configuration does not exceed the number of regions for the given table. 
+   * 
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public void limitNumReduceTasks(String table, JobConf job) 
+  throws IOException { 
+    HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    if (job.getNumReduceTasks() > regions)
+      job.setNumReduceTasks(regions);
+  }
+
+  /**
+   * Ensures that the given number of map tasks for the given job 
+   * configuration does not exceed the number of regions for the given table. 
+   * 
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public void limitNumMapTasks(String table, JobConf job) 
+  throws IOException { 
+    HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    if (job.getNumMapTasks() > regions)
+      job.setNumMapTasks(regions);
+  }
+
+  /**
+   * Sets the number of reduce tasks for the given job configuration to the 
+   * number of regions the given table has. 
+   * 
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public void setNumReduceTasks(String table, JobConf job) 
+  throws IOException { 
+    HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    job.setNumReduceTasks(regions);
+  }
+  
+  /**
+   * Sets the number of map tasks for the given job configuration to the 
+   * number of regions the given table has. 
+   * 
+   * @param table  The table to get the region count for.
+   * @param job  The current job configuration to adjust.
+   * @throws IOException When retrieving the table details fails.
+   */
+  public void setNumMapTasks(String table, JobConf job) 
+  throws IOException { 
+    HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+    int regions = outputTable.getRegionsInfo().size();
+    job.setNumMapTasks(regions);
+  }
+  
 }
\ No newline at end of file



Mime
View raw message