hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject svn commit: r1352220 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/regionserver/
Date Wed, 20 Jun 2012 17:33:20 GMT
Author: ramkrishna
Date: Wed Jun 20 17:33:19 2012
New Revision: 1352220

URL: http://svn.apache.org/viewvc?rev=1352220&view=rev
Log:
HBASE=6164 Correct the bug in block encoding usage in bulkload

Submitted by:	Anoop
Reviewed by:	Ted

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
Wed Jun 20 17:33:19 2012
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.io.encodi
 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * Controls what kind of data block encoding is used. If data block encoding is
@@ -33,7 +33,9 @@ import org.apache.hadoop.hbase.regionser
  */
 @InterfaceAudience.Private
 public interface HFileDataBlockEncoder {
-
+  /** Type of encoding used for data blocks in HFile. Stored in file info. */
+  byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+  
   /**
    * Converts a block from the on-disk format to the in-cache format. Called in
    * the following cases:
@@ -71,11 +73,11 @@ public interface HFileDataBlockEncoder {
   public boolean useEncodedScanner(boolean isCompaction);
 
   /**
-   * Save metadata in StoreFile which will be written to disk
-   * @param storeFileWriter writer for a given StoreFile
+   * Save metadata in HFile which will be written to disk
+   * @param writer writer for a given HFile
    * @exception IOException on disk problems
    */
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
+  public void saveMetadata(HFile.Writer writer)
       throws IOException;
 
   /** @return the on-disk data block encoding */

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
Wed Jun 20 17:33:19 2012
@@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.io.encodi
 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import com.google.common.base.Preconditions;
@@ -96,8 +95,7 @@ public class HFileDataBlockEncoderImpl i
   public static HFileDataBlockEncoder createFromFileInfo(
       FileInfo fileInfo, DataBlockEncoding preferredEncodingInCache)
       throws IOException {
-    byte[] dataBlockEncodingType =
-        fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
     if (dataBlockEncodingType == null) {
       return NoOpDataBlockEncoder.INSTANCE;
     }
@@ -128,10 +126,8 @@ public class HFileDataBlockEncoderImpl i
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
-      throws IOException {
-    storeFileWriter.appendFileInfo(StoreFile.DATA_BLOCK_ENCODING,
-        onDisk.getNameInBytes());
+  public void saveMetadata(HFile.Writer writer) throws IOException {
+    writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
Wed Jun 20 17:33:19 2012
@@ -334,6 +334,8 @@ public class HFileWriterV1 extends Abstr
     if (this.outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
Wed Jun 20 17:33:19 2012
@@ -358,6 +358,8 @@ public class HFileWriterV2 extends Abstr
     if (outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
Wed Jun 20 17:33:19 2012
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.io.encodi
 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 
 /**
  * Does not perform any kind of encoding/decoding.
@@ -71,7 +70,7 @@ public class NoOpDataBlockEncoder implem
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter) {
+  public void saveMetadata(HFile.Writer writer) {
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Wed Jun 20 17:33:19 2012
@@ -137,10 +137,6 @@ public class StoreFile extends SchemaCon
   /** Key for timestamp of earliest-put in metadata*/
   public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
 
-  /** Type of encoding used for data blocks in HFile. Stored in file info. */
-  public static final byte[] DATA_BLOCK_ENCODING =
-      Bytes.toBytes("DATA_BLOCK_ENCODING");
-
   // Make default block size for StoreFiles 8k while testing.  TODO: FIX!
   // Need to make it 8k for testing.
   public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -1208,9 +1204,6 @@ public class StoreFile extends SchemaCon
     }
 
     public void close() throws IOException {
-      // Save data block encoder metadata in the file info.
-      dataBlockEncoder.saveMetadata(this);
-
       boolean hasGeneralBloom = this.closeGeneralBloomFilter();
       boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1352220&r1=1352219&r2=1352220&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Wed Jun 20 17:33:19 2012
@@ -913,7 +913,7 @@ public class TestStoreFile extends HBase
     StoreFile.Reader reader = storeFile.createReader();
     
     Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
-    byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
 
     assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
   }



Mime
View raw message