hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jdcry...@apache.org
Subject svn commit: r707784 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/io/ src/java/org/apache/hadoop/hbase/mapred/ src/test/org/apache/hadoop/hbase/mapred/
Date Fri, 24 Oct 2008 23:24:10 GMT
Author: jdcryans
Date: Fri Oct 24 16:24:09 2008
New Revision: 707784

URL: http://svn.apache.org/viewvc?rev=707784&view=rev
Log:
HBASE-940   Make the TableOutputFormat batching-aware

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BatchUpdate.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=707784&r1=707783&r2=707784&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Oct 24 16:24:09 2008
@@ -65,6 +65,7 @@
                SubString operator (Clint Morgan via Stack)
    HBASE-937   Thrift getRow does not support specifying columns
                (Doğacan Güney via Stack)
+   HBASE-940   Make the TableOutputFormat batching-aware
     
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BatchUpdate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BatchUpdate.java?rev=707784&r1=707783&r2=707784&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BatchUpdate.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/io/BatchUpdate.java Fri Oct 24 16:24:09
2008
@@ -89,6 +89,17 @@
   public BatchUpdate(final String row, long timestamp){
     this(Bytes.toBytes(row), timestamp);
   }
+  
+  /**
+   * Recopy constructor
+   * @param buToCopy BatchUpdate to copy
+   */
+  public BatchUpdate(BatchUpdate buToCopy) {
+    this(buToCopy.getRow(), buToCopy.getTimestamp());
+    for(BatchOperation bo : buToCopy) {
+      this.put(bo.getColumn(), bo.getValue());
+    }
+  }
 
   /**
    * Initialize a BatchUpdate operation on a row with a specific timestamp.

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java?rev=707784&r1=707783&r2=707784&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java Fri
Oct 24 16:24:09 2008
@@ -24,7 +24,8 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.io.BatchUpdate;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java?rev=707784&r1=707783&r2=707784&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java Fri
Oct 24 16:24:09 2008
@@ -64,12 +64,17 @@
     }
 
     public void close(@SuppressWarnings("unused") Reporter reporter) {
-      // Nothing to do.
+      try {
+        m_table.flushCommits();
+      }
+      catch(IOException ioe) { 
+        LOG.error(ioe);
+      }
     }
 
     public void write(@SuppressWarnings("unused") ImmutableBytesWritable key,
         BatchUpdate value) throws IOException {
-      m_table.commit(value);
+      m_table.commit(new BatchUpdate(value));
     }
   }
   
@@ -91,6 +96,7 @@
       LOG.error(e);
       throw e;
     }
+    table.setAutoFlush(false);
     return new TableRecordWriter(table);
   }
 

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java?rev=707784&r1=707783&r2=707784&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java Fri
Oct 24 16:24:09 2008
@@ -135,7 +135,7 @@
       TableMapReduceUtil.initTableReduceJob(Bytes.toString(table.getTableName()),
         IdentityTableReduce.class, jobConf);
             
-      LOG.info("Started " + table.getTableName());
+      LOG.info("Started " + Bytes.toString(table.getTableName()));
       JobClient.runJob(jobConf);
       LOG.info("After map/reduce completion");
 



Mime
View raw message