hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1344561 - /hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
Date Thu, 31 May 2012 04:27:11 GMT
Author: stack
Date: Thu May 31 04:27:11 2012
New Revision: 1344561

URL: http://svn.apache.org/viewvc?rev=1344561&view=rev
Log:
HBASE-6040 Use block encoding and HBase handled checksum verification in bulk loading using
HFileOutputFormat

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java?rev=1344561&r1=1344560&r2=1344561&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
Thu May 31 04:27:11 2012
@@ -46,9 +46,14 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -75,6 +80,8 @@ public class HFileOutputFormat extends F
   static Log LOG = LogFactory.getLog(HFileOutputFormat.class);
   static final String COMPRESSION_CONF_KEY = "hbase.hfileoutputformat.families.compression";
   TimeRangeTracker trt = new TimeRangeTracker();
+  private static final String DATABLOCK_ENCODING_CONF_KEY = 
+     "hbase.mapreduce.hfileoutputformat.datablock.encoding";
 
   public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext
context)
   throws IOException, InterruptedException {
@@ -96,6 +103,22 @@ public class HFileOutputFormat extends F
 
     // create a map from column family to the compression algorithm
     final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf);
+    
+    String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY);
+    final HFileDataBlockEncoder encoder;
+    if (dataBlockEncodingStr == null) {
+      encoder = NoOpDataBlockEncoder.INSTANCE;
+    } else {
+      try {
+        encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding
+            .valueOf(dataBlockEncodingStr));
+      } catch (IllegalArgumentException ex) {
+        throw new RuntimeException(
+            "Invalid data block encoding type configured for the param "
+                + DATABLOCK_ENCODING_CONF_KEY + " : "
+                + dataBlockEncodingStr);
+      }
+    }
 
     return new RecordWriter<ImmutableBytesWritable, KeyValue>() {
       // Map of families to writers and how much has been output on the writer.
@@ -178,6 +201,9 @@ public class HFileOutputFormat extends F
             .withBlockSize(blocksize)
             .withCompression(compression)
             .withComparator(KeyValue.KEY_COMPARATOR)
+            .withDataBlockEncoder(encoder)
+            .withChecksumType(Store.getChecksumType(conf))
+            .withBytesPerChecksum(Store.getBytesPerChecksum(conf))
             .create();
         this.writers.put(family, wl);
         return wl;



Mime
View raw message