hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject svn commit: r1352221 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/io/hfile/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/regionserver/
Date Wed, 20 Jun 2012 17:34:52 GMT
Author: ramkrishna
Date: Wed Jun 20 17:34:51 2012
New Revision: 1352221

URL: http://svn.apache.org/viewvc?rev=1352221&view=rev
Log:
HBASE-6164 Correct the bug in block encoding usage in bulkload

Submitted by:Anoop	
Reviewed by:Ted	

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoder.java
Wed Jun 20 17:34:51 2012
@@ -20,7 +20,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -29,6 +29,9 @@ import org.apache.hadoop.hbase.util.Pair
  * should just return the unmodified block.
  */
 public interface HFileDataBlockEncoder {
+  /** Type of encoding used for data blocks in HFile. Stored in file info. */
+  public static final byte[] DATA_BLOCK_ENCODING = Bytes.toBytes("DATA_BLOCK_ENCODING");
+  
   /**
    * Converts a block from the on-disk format to the in-cache format. Called in
    * the following cases:
@@ -64,12 +67,11 @@ public interface HFileDataBlockEncoder {
   public boolean useEncodedScanner(boolean isCompaction);
 
   /**
-   * Save metadata in StoreFile which will be written to disk
-   * @param storeFileWriter writer for a given StoreFile
+   * Save metadata in HFile which will be written to disk
+   * @param writer writer for a given HFile
    * @exception IOException on disk problems
    */
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
-      throws IOException;
+  public void saveMetadata(HFile.Writer writer) throws IOException;
 
   /** @return the on-disk data block encoding */
   public DataBlockEncoding getEncodingOnDisk();

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
Wed Jun 20 17:34:51 2012
@@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.io.encodi
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.HFileBlock;
 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 
@@ -70,7 +69,7 @@ public class HFileDataBlockEncoderImpl i
     boolean hasPreferredCacheEncoding = preferredEncodingInCache != null
         && preferredEncodingInCache != DataBlockEncoding.NONE;
 
-    byte[] dataBlockEncodingType = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
     if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
       return NoOpDataBlockEncoder.INSTANCE;
     }
@@ -105,10 +104,8 @@ public class HFileDataBlockEncoderImpl i
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter)
-      throws IOException {
-    storeFileWriter.appendFileInfo(StoreFile.DATA_BLOCK_ENCODING,
-        onDisk.getNameInBytes());
+  public void saveMetadata(HFile.Writer writer) throws IOException {
+    writer.appendFileInfo(DATA_BLOCK_ENCODING, onDisk.getNameInBytes());
   }
 
   @Override

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
Wed Jun 20 17:34:51 2012
@@ -332,6 +332,8 @@ public class HFileWriterV1 extends Abstr
     if (this.outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
Wed Jun 20 17:34:51 2012
@@ -356,6 +356,8 @@ public class HFileWriterV2 extends Abstr
     if (outputStream == null) {
       return;
     }
+    // Save data block encoder metadata in the file info.
+    blockEncoder.saveMetadata(this);
     // Write out the end of the data blocks, then write meta data blocks.
     // followed by fileinfo, data block index and meta block index.
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java
Wed Jun 20 17:34:51 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.io.hfile
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -54,7 +53,7 @@ public class NoOpDataBlockEncoder implem
   }
 
   @Override
-  public void saveMetadata(StoreFile.Writer storeFileWriter) {
+  public void saveMetadata(HFile.Writer writer) {
   }
 
   @Override

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Wed Jun 20 17:34:51 2012
@@ -135,10 +135,6 @@ public class StoreFile extends SchemaCon
   /** Key for timestamp of earliest-put in metadata*/
   public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
 
-  /** Type of encoding used for data blocks in HFile. Stored in file info. */
-  public static final byte[] DATA_BLOCK_ENCODING =
-      Bytes.toBytes("DATA_BLOCK_ENCODING");
-
   // Make default block size for StoreFiles 8k while testing.  TODO: FIX!
   // Need to make it 8k for testing.
   public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -1205,9 +1201,6 @@ public class StoreFile extends SchemaCon
     }
 
     public void close() throws IOException {
-      // Save data block encoder metadata in the file info.
-      dataBlockEncoder.saveMetadata(this);
-
       boolean hasGeneralBloom = this.closeGeneralBloomFilter();
       boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
 

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1352221&r1=1352220&r2=1352221&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
(original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Wed Jun 20 17:34:51 2012
@@ -916,7 +916,7 @@ public class TestStoreFile extends HBase
     StoreFile.Reader reader = storeFile.createReader();
     
     Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
-    byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+    byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
 
     assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
   }



Mime
View raw message