hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1245291 [7/7] - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/io/ main/java/org/apache/hadoop/hbase/io/encoding/ main/java/org/apache/hadoop/h...
Date Fri, 17 Feb 2012 01:56:35 GMT
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java Fri Feb 17 01:56:33 2012
@@ -1,21 +1,18 @@
 /*
- * Copyright 2011 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
  */
 package org.apache.hadoop.hbase.regionserver;
 
@@ -25,9 +22,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
-import java.util.Set;
 import java.util.SortedSet;
-import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.ExecutionException;
@@ -54,16 +49,19 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
-import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.MD5Hash;
 import org.apache.hadoop.util.StringUtils;
@@ -73,6 +71,8 @@ import org.apache.hadoop.util.StringUtil
  */
 public class HFileReadWriteTest {
 
+  private static final String TABLE_NAME = "MyTable";
+
   private static enum Workload {
     MERGE("merge", "Merge the specified HFiles", 1, Integer.MAX_VALUE),
     RANDOM_READS("read", "Perform a random read benchmark on the given HFile",
@@ -136,11 +136,19 @@ public class HFileReadWriteTest {
   private Workload workload;
   private FileSystem fs;
   private Configuration conf;
+
+
+  private CacheConfig cacheConf;
   private List<String> inputFileNames;
   private Path outputDir;
   private int numReadThreads;
   private int durationSec;
 
+  private DataBlockEncoding dataBlockEncoding;
+  private boolean encodeInCacheOnly;
+  private HFileDataBlockEncoder dataBlockEncoder =
+      NoOpDataBlockEncoder.INSTANCE;
+
   private StoreFile.BloomType bloomType = StoreFile.BloomType.NONE;
   private int blockSize;
   private Compression.Algorithm compression = Compression.Algorithm.NONE;
@@ -158,8 +166,21 @@ public class HFileReadWriteTest {
   private SortedSet<String> keysRead = new ConcurrentSkipListSet<String>();
   private List<StoreFile> inputStoreFiles;
 
+  public static final String OPT_DATA_BLOCK_ENCODING =
+      HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
+  public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
+      "Encoding algorithm (e.g. prefix "
+          + "compression) to use for data blocks in the test column family, "
+          + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
+  public static final String OPT_ENCODE_IN_CACHE_ONLY =
+      "encode_in_cache_only";
+  public static final String OPT_ENCODE_IN_CACHE_ONLY_USAGE =
+      "If this is specified, data blocks will only be encoded in block " +
+      "cache but not on disk";
+
   public HFileReadWriteTest() {
     conf = HBaseConfiguration.create();
+    cacheConf = new CacheConfig(conf);
   }
 
   @SuppressWarnings("unchecked")
@@ -177,11 +198,15 @@ public class HFileReadWriteTest {
     options.addOption(BLOCK_SIZE_OPTION, true, "HFile block size" +
         Workload.MERGE.onlyUsedFor());
     options.addOption(DURATION_OPTION, true, "The amount of time to run the " +
-		"random read workload for" + Workload.RANDOM_READS.onlyUsedFor());
+        "random read workload for" + Workload.RANDOM_READS.onlyUsedFor());
     options.addOption(NUM_THREADS_OPTION, true, "The number of random " +
         "reader threads" + Workload.RANDOM_READS.onlyUsedFor());
     options.addOption(NUM_THREADS_OPTION, true, "The number of random " +
         "reader threads" + Workload.RANDOM_READS.onlyUsedFor());
+    options.addOption(OPT_DATA_BLOCK_ENCODING, true,
+        OPT_DATA_BLOCK_ENCODING_USAGE);
+    options.addOption(OPT_ENCODE_IN_CACHE_ONLY, false,
+        OPT_ENCODE_IN_CACHE_ONLY_USAGE);
     options.addOptionGroup(Workload.getOptionGroup());
 
     if (args.length == 0) {
@@ -233,6 +258,25 @@ public class HFileReadWriteTest {
           BLOOM_FILTER_OPTION));
     }
 
+    encodeInCacheOnly =
+        cmdLine.hasOption(OPT_ENCODE_IN_CACHE_ONLY);
+
+    if (cmdLine.hasOption(OPT_DATA_BLOCK_ENCODING)) {
+      dataBlockEncoding = DataBlockEncoding.valueOf(
+          cmdLine.getOptionValue(OPT_DATA_BLOCK_ENCODING));
+      // Optionally encode on disk, always encode in cache.
+      dataBlockEncoder = new HFileDataBlockEncoderImpl(
+          encodeInCacheOnly ? DataBlockEncoding.NONE : dataBlockEncoding,
+          dataBlockEncoding);
+    } else {
+      if (encodeInCacheOnly) {
+        LOG.error("The -" + OPT_ENCODE_IN_CACHE_ONLY +
+            " option does not make sense without -" +
+            OPT_DATA_BLOCK_ENCODING);
+        return false;
+      }
+    }
+
     blockSize = conf.getInt("hfile.min.blocksize.size", 65536);
     if (cmdLine.hasOption(BLOCK_SIZE_OPTION))
       blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION));
@@ -257,6 +301,7 @@ public class HFileReadWriteTest {
       numReadThreads = Integer.parseInt(
           cmdLine.getOptionValue(NUM_THREADS_OPTION));
     }
+
     Collections.sort(inputFileNames);
 
     return true;
@@ -292,7 +337,7 @@ public class HFileReadWriteTest {
     if (outputDir != null &&
         (!fs.exists(outputDir) || !fs.getFileStatus(outputDir).isDir())) {
       LOG.error(outputDir.toString() + " does not exist or is not a " +
-		"directory");
+          "directory");
       return false;
     }
 
@@ -304,23 +349,27 @@ public class HFileReadWriteTest {
 
     List<StoreFileScanner> scanners =
         StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false,
-            false, true);
+            false);
 
     HColumnDescriptor columnDescriptor = new HColumnDescriptor(
         HFileReadWriteTest.class.getSimpleName());
     columnDescriptor.setBlocksize(blockSize);
     columnDescriptor.setBloomFilterType(bloomType);
     columnDescriptor.setCompressionType(compression);
+
+    columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
     HRegionInfo regionInfo = new HRegionInfo();
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
     HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, null);
     Store store = new Store(outputDir, region, columnDescriptor, fs, conf);
 
-    StoreFile.Writer writer = StoreFile.createWriter(fs, outputDir,
-        blockSize,
-        compression, KeyValue.COMPARATOR, this.conf, new CacheConfig(this.conf),
-        bloomType, maxKeyCount);
+    StoreFile.Writer writer =
+        StoreFile.createWriter(fs, outputDir, blockSize, compression,
+            dataBlockEncoder, KeyValue.COMPARATOR, this.conf,
+            new CacheConfig(conf), bloomType, BloomFilterFactory.getErrorRate(conf),
+            maxKeyCount, null);
 
-    StatisticsPrinter statsPrinter = new StatisticsPrinter(getHFileReaders());
+    StatisticsPrinter statsPrinter = new StatisticsPrinter();
     statsPrinter.startThread();
 
     try {
@@ -426,8 +475,8 @@ public class HFileReadWriteTest {
       throws IOException {
     // We are passing the ROWCOL Bloom filter type, but StoreFile will still
     // use the Bloom filter type specified in the HFile.
-    return new StoreFile(fs, filePath, conf, new CacheConfig(this.conf),
-        StoreFile.BloomType.ROWCOL);
+    return new StoreFile(fs, filePath, conf, cacheConf,
+        StoreFile.BloomType.ROWCOL, dataBlockEncoder);
   }
 
   public static int charToHex(int c) {
@@ -590,12 +639,7 @@ public class HFileReadWriteTest {
 
     private volatile boolean stopRequested;
     private volatile Thread thread;
-    private Set<String> fsBlockReadMetrics = new TreeSet<String>();
-    private boolean isCompaction;
-
-    public StatisticsPrinter(HFile.Reader reader) {
-      this(new HFile.Reader[] { reader });
-    }
+    private long totalSeekAndReads, totalPositionalReads;
 
     /**
      * Run the statistics collector in a separate thread without an executor.
@@ -613,19 +657,6 @@ public class HFileReadWriteTest {
       }.start();
     }
 
-    public StatisticsPrinter(HFile.Reader[] readers) {
-      isCompaction = workload == Workload.MERGE;
-      for (HFile.Reader reader : readers) {
-        fsBlockReadMetrics.add(
-            SchemaMetrics.ALL_SCHEMA_METRICS.getBlockMetricName(
-                BlockType.BlockCategory.ALL_CATEGORIES, isCompaction,
-                SchemaMetrics.BlockMetricType.READ_COUNT));
-      }
-
-      LOG.info("Using the following metrics for the number of data blocks " +
-          "read: " + fsBlockReadMetrics);
-    }
-
     @Override
     public Boolean call() throws Exception {
       LOG.info("Starting statistics printer");
@@ -658,13 +689,19 @@ public class HFileReadWriteTest {
       double kvPerSec = kvCount / timeSec;
       long bytes = totalBytes.get();
       double bytesPerSec = bytes / timeSec;
-      long blocks = getTotalBlocksRead();
-      double blkReadPerSec = blocks / timeSec;
 
-      double seekReadPerSec = HFileBlock.getNumSeekAndReadOperations() /
-          timeSec;
-      double preadPerSec = HFileBlock.getNumPositionalReadOperations() /
-          timeSec;
+      // readOps and preadOps counters get reset on access, so we have to
+      // accumulate them here. HRegion metrics publishing thread should not
+      // be running in this tool, so no one else should be resetting these
+      // metrics.
+      totalSeekAndReads += HFile.getReadOps();
+      totalPositionalReads += HFile.getPreadOps();
+      long totalBlocksRead = totalSeekAndReads + totalPositionalReads;
+
+      double blkReadPerSec = totalBlocksRead / timeSec;
+
+      double seekReadPerSec = totalSeekAndReads / timeSec;
+      double preadPerSec = totalPositionalReads / timeSec;
 
       boolean isRead = workload == Workload.RANDOM_READS;
 
@@ -677,7 +714,7 @@ public class HFileReadWriteTest {
       sb.append(", blk/sec: " + (long) blkReadPerSec);
       sb.append(", total KV: " + numKV);
       sb.append(", total bytes: " + totalBytes);
-      sb.append(", total blk: " + blocks);
+      sb.append(", total blk: " + totalBlocksRead);
 
       sb.append(", seekRead/sec: " + (long) seekReadPerSec);
       sb.append(", pread/sec: " + (long) preadPerSec);
@@ -688,13 +725,6 @@ public class HFileReadWriteTest {
       LOG.info(sb.toString());
     }
 
-    private long getTotalBlocksRead() {
-      long totalBlocksRead = 0;
-      for (String metric : fsBlockReadMetrics)
-        return HRegion.getNumericMetric(metric);
-      return totalBlocksRead;
-    }
-
     public void requestStop() {
       stopRequested = true;
       if (thread != null)
@@ -750,7 +780,7 @@ public class HFileReadWriteTest {
       boolean pread = true;
       for (int i = 0; i < numReadThreads; ++i)
         ecs.submit(new RandomReader(i, reader, pread));
-      ecs.submit(new StatisticsPrinter(reader.getHFileReader()));
+      ecs.submit(new StatisticsPrinter());
       Future<Boolean> result;
       while (true) {
         try {
@@ -780,9 +810,10 @@ public class HFileReadWriteTest {
       storeFile.closeReader(true);
       exec.shutdown();
 
-      LruBlockCache c = (LruBlockCache) new CacheConfig(conf).getBlockCache();
-      if (c != null)
+      BlockCache c = cacheConf.getBlockCache();
+      if (c != null) {
         c.shutdown();
+      }
     }
     LOG.info("Worker threads completed: " + numCompleted);
     LOG.info("Worker threads failed: " + numFailed);
@@ -824,4 +855,4 @@ public class HFileReadWriteTest {
     }
   }
 
-}
+}
\ No newline at end of file

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java Fri Feb 17 01:56:33 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.HConstant
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -68,7 +69,7 @@ public class TestCompactSelection extend
     this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
     this.conf.setInt("hbase.hstore.compaction.min", minFiles);
     this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
-    this.conf.setLong("hbase.hregion.memstore.flush.size", minSize);
+    this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
     this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
     this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);
 
@@ -100,7 +101,8 @@ public class TestCompactSelection extend
     MockStoreFile(long length, boolean isRef) throws IOException {
       super(TEST_UTIL.getTestFileSystem(), TEST_FILE,
             TEST_UTIL.getConfiguration(),
-            new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE);
+            new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE,
+            NoOpDataBlockEncoder.INSTANCE);
       this.length = length;
       this.isRef  = isRef;
     }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Fri Feb 17 01:56:33 2012
@@ -19,12 +19,15 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
-
-import static org.junit.Assert.fail;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -36,21 +39,20 @@ import org.apache.hadoop.hbase.HBaseTest
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.spy;
 
 
 /**
@@ -59,6 +61,7 @@ import static org.mockito.Mockito.spy;
 public class TestCompaction extends HBaseTestCase {
   static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
   private HRegion r = null;
+  private HTableDescriptor htd = null;
   private Path compactionDir = null;
   private Path regionCompactionDir = null;
   private static final byte [] COLUMN_FAMILY = fam1;
@@ -75,9 +78,9 @@ public class TestCompaction extends HBas
     super();
 
     // Set cache flush size to 1MB
-    conf.setInt("hbase.hregion.memstore.flush.size", 1024*1024);
-    conf.setInt("hbase.hregion.memstore.block.multiplier", 10);
-    this.cluster = null;
+    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
+    conf.setInt("hbase.hregion.memstore.block.multiplier", 100);
+
     compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
 
     firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
@@ -98,7 +101,7 @@ public class TestCompaction extends HBas
     this.conf.set(HConstants.HBASE_DIR,
       this.cluster.getFileSystem().getHomeDirectory().toString());
     super.setUp();
-    HTableDescriptor htd = createTableDescriptor(getName());
+    this.htd = createTableDescriptor(getName());
     this.r = createNewHRegion(htd, null, null);
   }
 
@@ -153,6 +156,42 @@ public class TestCompaction extends HBas
    * @throws Exception
    */
   public void testMajorCompaction() throws Exception {
+    majorCompaction();
+  }
+
+  public void testDataBlockEncodingInCacheOnly() throws Exception {
+    majorCompactionWithDataBlockEncoding(true);
+  }
+
+  public void testDataBlockEncodingEverywhere() throws Exception {
+    majorCompactionWithDataBlockEncoding(false);
+  }
+
+  public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly)
+      throws Exception {
+    Map<Store, HFileDataBlockEncoder> replaceBlockCache =
+        new HashMap<Store, HFileDataBlockEncoder>();
+    for (Entry<byte[], Store> pair : r.getStores().entrySet()) {
+      Store store = pair.getValue();
+      HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
+      replaceBlockCache.put(pair.getValue(), blockEncoder);
+      final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
+      final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE :
+          inCache;
+      store.setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(
+          onDisk, inCache));
+    }
+
+    majorCompaction();
+
+    // restore settings
+    for (Entry<Store, HFileDataBlockEncoder> entry :
+        replaceBlockCache.entrySet()) {
+      entry.getKey().setDataBlockEncoderInTest(entry.getValue());
+    }
+  }
+
+  private void majorCompaction() throws Exception {
     createStoreFile(r);
     for (int i = 0; i < compactionThreshold; i++) {
       createStoreFile(r);
@@ -176,14 +215,20 @@ public class TestCompaction extends HBas
     secondRowBytes[START_KEY_BYTES.length - 1]++;
 
     // Always 3 versions if that is what max versions is.
-    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
-    assertEquals(compactionThreshold, result.size());
+    result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).
+        setMaxVersions(100), null);
+    LOG.debug("Row " + Bytes.toStringBinary(secondRowBytes) + " after " +
+        "initial compaction: " + result);
+    assertEquals("Invalid number of versions of row "
+        + Bytes.toStringBinary(secondRowBytes) + ".", compactionThreshold,
+        result.size());
 
     // Now add deletes to memstore and then flush it.
     // That will put us over
     // the compaction threshold of 3 store files.  Compacting these store files
     // should result in a compacted store file that has no references to the
     // deleted row.
+    LOG.debug("Adding deletes to memstore and flushing");
     Delete delete = new Delete(secondRowBytes, System.currentTimeMillis(), null);
     byte [][] famAndQf = {COLUMN_FAMILY, null};
     delete.deleteFamily(famAndQf[0]);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Fri Feb 17 01:56:33 2012
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Sc
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
@@ -192,7 +193,8 @@ public class TestCompoundBloomFilter {
 
   private void readStoreFile(int t, BloomType bt, List<KeyValue> kvs,
       Path sfPath) throws IOException {
-    StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt);
+    StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt,
+        NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader r = sf.createReader();
     final boolean pread = true; // does not really matter
     StoreFileScanner scanner = r.getStoreFileScanner(true, pread);
@@ -292,8 +294,8 @@ public class TestCompoundBloomFilter {
     cacheConf = new CacheConfig(conf);
 
     StoreFile.Writer w = StoreFile.createWriter(fs,
-        HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null, conf,
-        cacheConf, bt, 0);
+        HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null,
+        conf, cacheConf, bt, 0);
 
     assertTrue(w.hasGeneralBloom());
     assertTrue(w.getGeneralBloomWriter() instanceof CompoundBloomFilterWriter);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Fri Feb 17 01:56:33 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.HB
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
@@ -77,7 +78,9 @@ public class TestFSErrorsExposed {
         writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
 
     StoreFile sf = new StoreFile(fs, writer.getPath(),
-        util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE);
+        util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE,
+        NoOpDataBlockEncoder.INSTANCE);
+
     StoreFile.Reader reader = sf.createReader();
     HFileScanner scanner = reader.getScanner(false, true);
 
@@ -120,7 +123,8 @@ public class TestFSErrorsExposed {
         writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
 
     StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(),
-        cacheConf, BloomType.NONE);
+        cacheConf, BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
+
     List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(
         Collections.singletonList(sf), false, true, false);
     KeyValueScanner scanner = scanners.get(0);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java Fri Feb 17 01:56:33 2012
@@ -19,7 +19,9 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,6 +35,7 @@ import java.util.Random;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -44,8 +47,10 @@ import org.apache.hadoop.hbase.KeyValueT
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockType;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
@@ -109,8 +114,9 @@ public class TestMultiColumnScanner {
   private final static HBaseTestingUtility TEST_UTIL =
     new HBaseTestingUtility();
 
-  private Compression.Algorithm comprAlgo;
-  private StoreFile.BloomType bloomType;
+  private final Compression.Algorithm comprAlgo;
+  private final StoreFile.BloomType bloomType;
+  private final DataBlockEncoding dataBlockEncoding;
 
   private long lastBlocksRead;
   private long lastCacheHits;
@@ -133,13 +139,23 @@ public class TestMultiColumnScanner {
 
   @Parameters
   public static final Collection<Object[]> parameters() {
-    return HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS;
+    List<Object[]> parameters = new ArrayList<Object[]>();
+    for (Object[] bloomAndCompressionParams :
+        HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS) {
+      for (boolean useDataBlockEncoding : new boolean[]{false, true}) {
+        parameters.add(ArrayUtils.add(bloomAndCompressionParams,
+            useDataBlockEncoding));
+      }
+    }
+    return parameters;
   }
 
   public TestMultiColumnScanner(Compression.Algorithm comprAlgo,
-      StoreFile.BloomType bloomType) {
+      StoreFile.BloomType bloomType, boolean useDataBlockEncoding) {
     this.comprAlgo = comprAlgo;
     this.bloomType = bloomType;
+    this.dataBlockEncoding = useDataBlockEncoding ? DataBlockEncoding.PREFIX :
+        DataBlockEncoding.NONE;
   }
 
   private long getBlocksRead() {
@@ -177,8 +193,9 @@ public class TestMultiColumnScanner {
 
   @Test
   public void testMultiColumnScanner() throws IOException {
-    HRegion region = createRegion(TABLE_NAME, comprAlgo, bloomType,
-        MAX_VERSIONS);
+    HRegion region = TEST_UTIL.createTestRegion(TABLE_NAME, FAMILY, comprAlgo,
+        bloomType, MAX_VERSIONS, HFile.DEFAULT_BLOCKSIZE,
+        dataBlockEncoding, true);
     List<String> rows = sequentialStrings("row", NUM_ROWS);
     List<String> qualifiers = sequentialStrings("qual", NUM_COLUMNS);
     List<KeyValue> kvs = new ArrayList<KeyValue>();

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java Fri Feb 17 01:56:33 2012
@@ -20,7 +20,9 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueTestUtil;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Before;
@@ -76,8 +78,10 @@ public class TestScanWithBloomError {
 
   @Test
   public void testThreeStoreFiles() throws IOException {
-    region = createRegion(TABLE_NAME, Compression.Algorithm.GZ, bloomType,
-        MAX_VERSIONS);
+    region = TEST_UTIL.createTestRegion(TABLE_NAME, FAMILY,
+        Compression.Algorithm.GZ, bloomType,
+        TestMultiColumnScanner.MAX_VERSIONS, HFile.DEFAULT_BLOCKSIZE,
+        DataBlockEncoding.NONE, true);
     createStoreFile(new int[] {1, 2, 6});
     createStoreFile(new int[] {1, 2, 3, 7});
     createStoreFile(new int[] {1, 9});

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java Fri Feb 17 01:56:33 2012
@@ -43,7 +43,9 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.After;
 import org.junit.Before;
@@ -112,6 +114,9 @@ public class TestSeekOptimizations {
 
   private long totalSeekDiligent, totalSeekLazy;
 
+  private final static HBaseTestingUtility TEST_UTIL =
+      new HBaseTestingUtility();
+
   @Parameters
   public static final Collection<Object[]> parameters() {
     return HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS;
@@ -131,9 +136,9 @@ public class TestSeekOptimizations {
 
   @Test
   public void testMultipleTimestampRanges() throws IOException {
-    region = TestMultiColumnScanner.createRegion(
-        TestSeekOptimizations.class.getName(), comprAlgo, bloomType,
-        Integer.MAX_VALUE);
+    region = TEST_UTIL.createTestRegion(TestSeekOptimizations.class.getName(),
+        FAMILY, comprAlgo, bloomType, Integer.MAX_VALUE,
+        HFile.DEFAULT_BLOCKSIZE, DataBlockEncoding.NONE, true);
 
     // Delete the given timestamp and everything before.
     final long latestDelTS = USE_MANY_STORE_FILES ? 1397 : -1;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Fri Feb 17 01:56:33 2012
@@ -36,15 +36,21 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
 import org.apache.hadoop.hbase.util.BloomFilterFactory;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -100,7 +106,7 @@ public class TestStoreFile extends HBase
       conf, cacheConf);
     writeStoreFile(writer);
     checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
-        StoreFile.BloomType.NONE));
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE));
   }
 
   private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
@@ -141,7 +147,7 @@ public class TestStoreFile extends HBase
         conf, cacheConf);
     writeStoreFile(writer);
     StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader reader = hsf.createReader();
     // Split on a row, not in middle of row.  Midkey returned by reader
     // may be in middle of row.  Create new one with empty column and
@@ -153,7 +159,7 @@ public class TestStoreFile extends HBase
     // Make a reference
     Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top);
     StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     // Now confirm that I can read from the reference and that it only gets
     // keys from top half of the file.
     HFileScanner s = refHsf.createReader().getScanner(false, false);
@@ -189,10 +195,12 @@ public class TestStoreFile extends HBase
     Path bottomPath = StoreFile.split(this.fs, bottomDir,
       f, midRow, Range.bottom);
     // Make readers on top and bottom.
-    StoreFile.Reader top = new StoreFile(this.fs, topPath, conf, cacheConf,
-        StoreFile.BloomType.NONE).createReader();
-    StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
-        StoreFile.BloomType.NONE).createReader();
+    StoreFile.Reader top =
+        new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE,
+            NoOpDataBlockEncoder.INSTANCE).createReader();
+    StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath,
+        conf, cacheConf, BloomType.NONE,
+        NoOpDataBlockEncoder.INSTANCE).createReader();
     ByteBuffer previous = null;
     LOG.info("Midkey: " + midKV.toString());
     ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey);
@@ -248,9 +256,11 @@ public class TestStoreFile extends HBase
       bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
         Range.bottom);
       top = new StoreFile(this.fs, topPath, conf, cacheConf,
-          StoreFile.BloomType.NONE).createReader();
+          StoreFile.BloomType.NONE,
+          NoOpDataBlockEncoder.INSTANCE).createReader();
       bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
-          StoreFile.BloomType.NONE).createReader();
+          StoreFile.BloomType.NONE,
+          NoOpDataBlockEncoder.INSTANCE).createReader();
       bottomScanner = bottom.getScanner(false, false);
       int count = 0;
       while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -293,9 +303,11 @@ public class TestStoreFile extends HBase
       bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
         Range.bottom);
       top = new StoreFile(this.fs, topPath, conf, cacheConf,
-          StoreFile.BloomType.NONE).createReader();
+          StoreFile.BloomType.NONE,
+          NoOpDataBlockEncoder.INSTANCE).createReader();
       bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
-          StoreFile.BloomType.NONE).createReader();
+          StoreFile.BloomType.NONE,
+          NoOpDataBlockEncoder.INSTANCE).createReader();
       first = true;
       bottomScanner = bottom.getScanner(false, false);
       while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -353,7 +365,8 @@ public class TestStoreFile extends HBase
     }
     writer.close();
 
-    StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf);
+    StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+        DataBlockEncoding.NONE);
     reader.loadFileInfo();
     reader.loadBloomfilter();
     StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
@@ -391,10 +404,10 @@ public class TestStoreFile extends HBase
 
     // write the file
     Path f = new Path(ROOT_DIR, getName());
-    StoreFile.Writer writer = new StoreFile.Writer(fs, f,
-        StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
-        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
-
+    StoreFile.Writer writer =
+        new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL,
+            HFile.DEFAULT_COMPRESSION_ALGORITHM, conf, cacheConf,
+            KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
     bloomWriteRead(writer, fs);
   }
 
@@ -410,7 +423,8 @@ public class TestStoreFile extends HBase
     Path f = new Path(ROOT_DIR, getName());
     StoreFile.Writer writer = new StoreFile.Writer(fs, f,
         StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
-        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000);
+        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
+        2000);
 
     // add delete family
     long now = System.currentTimeMillis();
@@ -422,8 +436,8 @@ public class TestStoreFile extends HBase
     }
     writer.close();
 
-    StoreFile.Reader reader = new StoreFile.Reader(fs, writer.getPath(),
-        cacheConf);
+    StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+        DataBlockEncoding.NONE);
     reader.loadFileInfo();
     reader.loadBloomfilter();
 
@@ -495,7 +509,8 @@ public class TestStoreFile extends HBase
       }
       writer.close();
 
-      StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf);
+      StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+          DataBlockEncoding.NONE);
       reader.loadFileInfo();
       reader.loadBloomfilter();
       StoreFileScanner scanner = reader.getStoreFileScanner(true, true);
@@ -549,7 +564,8 @@ public class TestStoreFile extends HBase
     // this should not create a bloom because the max keys is too small
     StoreFile.Writer writer = new StoreFile.Writer(fs, f,
         StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
-        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
+        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW,
+        2000);
     assertFalse(writer.hasGeneralBloom());
     writer.close();
     fs.delete(f, true);
@@ -677,7 +693,7 @@ public class TestStoreFile extends HBase
     writer.close();
 
     StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader reader = hsf.createReader();
     StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
     TreeSet<byte[]> columns = new TreeSet<byte[]>();
@@ -721,7 +737,7 @@ public class TestStoreFile extends HBase
     Path pathCowOff = new Path(baseDir, "123456789");
     StoreFile.Writer writer = writeStoreFile(conf, cacheConf, pathCowOff, 3);
     StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     LOG.debug(hsf.getPath().toString());
 
     // Read this file, we should see 3 misses
@@ -743,7 +759,7 @@ public class TestStoreFile extends HBase
     Path pathCowOn = new Path(baseDir, "123456788");
     writer = writeStoreFile(conf, cacheConf, pathCowOn, 3);
     hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
 
     // Read this file, we should see 3 hits
     reader = hsf.createReader();
@@ -759,14 +775,14 @@ public class TestStoreFile extends HBase
 
     // Let's read back the two files to ensure the blocks exactly match
     hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader readerOne = hsf.createReader();
     readerOne.loadFileInfo();
     StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, true,
         false);
     scannerOne.seek(KeyValue.LOWESTKEY);
     hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     StoreFile.Reader readerTwo = hsf.createReader();
     readerTwo.loadFileInfo();
     StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, true,
@@ -798,7 +814,7 @@ public class TestStoreFile extends HBase
     conf.setBoolean("hbase.rs.evictblocksonclose", true);
     cacheConf = new CacheConfig(conf);
     hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     reader = hsf.createReader();
     reader.close(cacheConf.shouldEvictOnClose());
 
@@ -812,7 +828,7 @@ public class TestStoreFile extends HBase
     conf.setBoolean("hbase.rs.evictblocksonclose", false);
     cacheConf = new CacheConfig(conf);
     hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
-        StoreFile.BloomType.NONE);
+        StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
     reader = hsf.createReader();
     reader.close(cacheConf.shouldEvictOnClose());
 
@@ -839,7 +855,8 @@ public class TestStoreFile extends HBase
     int blockSize = totalSize / numBlocks;
     StoreFile.Writer writer = new StoreFile.Writer(fs, path, blockSize,
         HFile.DEFAULT_COMPRESSION_ALGORITHM,
-        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000);
+        conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
+        2000);
     // We'll write N-1 KVs to ensure we don't write an extra block
     kvs.remove(kvs.size()-1);
     for (KeyValue kv : kvs) {
@@ -849,4 +866,40 @@ public class TestStoreFile extends HBase
     writer.close();
     return writer;
   }
+
+  /**
+   * Check if data block encoding information is saved correctly in HFile's
+   * file info.
+   */
+  public void testDataBlockEncodingMetaData() throws IOException {
+    Path dir = new Path(new Path(this.testDir, "regionname"), "familyname");
+    Path path = new Path(dir, "1234567890");
+
+    DataBlockEncoding dataBlockEncoderAlgo =
+        DataBlockEncoding.FAST_DIFF;
+    HFileDataBlockEncoder dataBlockEncoder =
+        new HFileDataBlockEncoderImpl(
+            dataBlockEncoderAlgo,
+            dataBlockEncoderAlgo);
+    cacheConf = new CacheConfig(conf);
+    StoreFile.Writer writer = new StoreFile.Writer(fs,
+        path, HFile.DEFAULT_BLOCKSIZE,
+        HFile.DEFAULT_COMPRESSION_ALGORITHM,
+        dataBlockEncoder,
+        conf,
+        cacheConf,
+        KeyValue.COMPARATOR,
+        StoreFile.BloomType.NONE,
+        HColumnDescriptor.DEFAULT_BLOOMFILTER_ERROR_RATE, 2000, null);
+    writer.close();
+
+    StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
+        cacheConf, BloomType.NONE, dataBlockEncoder);
+    StoreFile.Reader reader = storeFile.createReader();
+
+    Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
+    byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+
+    assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
+  }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java Fri Feb 17 01:56:33 2012
@@ -108,7 +108,7 @@ public class TestLogRolling extends HBas
     conf.setInt("hbase.hregion.memstore.optionalflushcount", 2);
 
     // We flush the cache after every 8192 bytes
-    conf.setInt("hbase.hregion.memstore.flush.size", 8192);
+    conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 8192);
 
     // Increase the amount of time between client retries
     conf.setLong("hbase.client.pause", 15 * 1000);

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java?rev=1245291&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java Fri Feb 17 01:56:33 2012
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.Random;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+
+/**
+ * A generator of random keys and values for load testing. Keys are generated
+ * by converting numeric indexes to strings and prefixing them with an MD5
+ * hash. Values are generated by selecting value size in the configured range
+ * and generating a pseudo-random sequence of bytes seeded by key, column
+ * qualifier, and value size.
+ * <p>
+ * Not thread-safe, so a separate instance is needed for every writer thread/
+ */
+public class LoadTestKVGenerator {
+
+  /** A random number generator for determining value size */
+  private Random randomForValueSize = new Random();
+
+  private final int minValueSize;
+  private final int maxValueSize;
+
+  public LoadTestKVGenerator(int minValueSize, int maxValueSize) {
+    if (minValueSize <= 0 || maxValueSize <= 0) {
+      throw new IllegalArgumentException("Invalid min/max value sizes: " +
+          minValueSize + ", " + maxValueSize);
+    }
+    this.minValueSize = minValueSize;
+    this.maxValueSize = maxValueSize;
+  }
+
+  /**
+   * Verifies that the given byte array is the same as what would be generated
+   * for the given row key and qualifier. We are assuming that the value size
+   * is correct, and only verify the actual bytes. However, if the min/max
+   * value sizes are set sufficiently high, an accidental match should be
+   * extremely improbable.
+   */
+  public static boolean verify(String rowKey, String qual, byte[] value) {
+    byte[] expectedData = getValueForRowColumn(rowKey, qual, value.length);
+    return Bytes.equals(expectedData, value);
+  }
+
+  /**
+   * Converts the given key to string, and prefixes it with the MD5 hash of
+   * the index's string representation.
+   */
+  public static String md5PrefixedKey(long key) {
+    String stringKey = Long.toString(key);
+    String md5hash = MD5Hash.getMD5AsHex(Bytes.toBytes(stringKey));
+
+    // flip the key to randomize
+    return md5hash + "-" + stringKey;
+  }
+
+  /**
+   * Generates a value for the given key index and column qualifier. Size is
+   * selected randomly in the configured range. The generated value depends
+   * only on the combination of the key, qualifier, and the selected value
+   * size. This allows to verify the actual value bytes when reading, as done
+   * in {@link #verify(String, String, byte[])}.
+   */
+  public byte[] generateRandomSizeValue(long key, String qual) {
+    String rowKey = md5PrefixedKey(key);
+    int dataSize = minValueSize + randomForValueSize.nextInt(
+        Math.abs(maxValueSize - minValueSize));
+    return getValueForRowColumn(rowKey, qual, dataSize);
+  }
+
+  /**
+   * Generates random bytes of the given size for the given row and column
+   * qualifier. The random seed is fully determined by these parameters.
+   */
+  private static byte[] getValueForRowColumn(String rowKey, String qual,
+      int dataSize) {
+    Random seededRandom = new Random(rowKey.hashCode() + qual.hashCode() +
+        dataSize);
+    byte[] randomBytes = new byte[dataSize];
+    seededRandom.nextBytes(randomBytes);
+    return randomBytes;
+  }
+
+  public static byte[] longToByteArrayKey(long rowKey) {
+    return md5PrefixedKey(rowKey).getBytes();
+  }
+
+}

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java?rev=1245291&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java Fri Feb 17 01:56:33 2012
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.io.WritableUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestByteBufferUtils {
+
+  private byte[] array;
+
+  /**
+   * Create an array with sample data.
+   */
+  @Before
+  public void setUp() {
+    array = new byte[8];
+    for (int i = 0; i < array.length; ++i) {
+      array[i] = (byte) ('a' + i);
+    }
+  }
+
+  private static final int MAX_VLONG_LENGTH = 9;
+  private static final Collection<Long> testNumbers;
+
+  private static void addNumber(Set<Long> a, long l) {
+    if (l != Long.MIN_VALUE) {
+      a.add(l - 1);
+    }
+    a.add(l);
+    if (l != Long.MAX_VALUE) {
+      a.add(l + 1);
+    }
+    for (long divisor = 3; divisor <= 10; ++divisor) {
+      for (long delta = -1; delta <= 1; ++delta) {
+        a.add(l / divisor + delta);
+      }
+    }
+  }
+
+  static {
+    SortedSet<Long> a = new TreeSet<Long>();
+    for (int i = 0; i <= 63; ++i) {
+      long v = (-1L) << i;
+      assertTrue(v < 0);
+      addNumber(a, v);
+      v = (1L << i) - 1;
+      assertTrue(v >= 0);
+      addNumber(a, v);
+    }
+
+    testNumbers = Collections.unmodifiableSet(a);
+    System.err.println("Testing variable-length long serialization using: "
+        + testNumbers + " (count: " + testNumbers.size() + ")");
+    assertEquals(1753, testNumbers.size());
+    assertEquals(Long.MIN_VALUE, a.first().longValue());
+    assertEquals(Long.MAX_VALUE, a.last().longValue());
+  }
+
+  @Test
+  public void testReadWriteVLong() {
+    for (long l : testNumbers) {
+      ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH);
+      ByteBufferUtils.writeVLong(b, l);
+      b.flip();
+      assertEquals(l, ByteBufferUtils.readVLong(b));
+    }
+  }
+
+  @Test
+  public void testConsistencyWithHadoopVLong() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    for (long l : testNumbers) {
+      baos.reset();
+      ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH);
+      ByteBufferUtils.writeVLong(b, l);
+      String bufStr = Bytes.toStringBinary(b.array(),
+          b.arrayOffset(), b.position());
+      WritableUtils.writeVLong(dos, l);
+      String baosStr = Bytes.toStringBinary(baos.toByteArray());
+      assertEquals(baosStr, bufStr);
+    }
+  }
+
+  /**
+   * Test copying to stream from buffer.
+   */
+  @Test
+  public void testMoveBufferToStream() {
+    final int arrayOffset = 7;
+    final int initialPosition = 10;
+    final int endPadding = 5;
+    byte[] arrayWrapper =
+        new byte[arrayOffset + initialPosition + array.length + endPadding];
+    System.arraycopy(array, 0, arrayWrapper,
+        arrayOffset + initialPosition, array.length);
+    ByteBuffer buffer = ByteBuffer.wrap(arrayWrapper, arrayOffset,
+        initialPosition + array.length).slice();
+    assertEquals(initialPosition + array.length, buffer.limit());
+    assertEquals(0, buffer.position());
+    buffer.position(initialPosition);
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    try {
+      ByteBufferUtils.moveBufferToStream(bos, buffer, array.length);
+    } catch (IOException e) {
+      fail("IOException in testCopyToStream()");
+    }
+    assertArrayEquals(array, bos.toByteArray());
+    assertEquals(initialPosition + array.length, buffer.position());
+  }
+
+  /**
+   * Test copying to stream from buffer with offset.
+   * @throws IOException On test failure.
+   */
+  @Test
+  public void testCopyToStreamWithOffset() throws IOException {
+    ByteBuffer buffer = ByteBuffer.wrap(array);
+
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+
+    ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2,
+        array.length / 2);
+
+    byte[] returnedArray = bos.toByteArray();
+    for (int i = 0; i < array.length / 2; ++i) {
+      int pos = array.length / 2 + i;
+      assertEquals(returnedArray[i], array[pos]);
+    }
+  }
+
+  /**
+   * Test copying data from stream.
+   * @throws IOException On test failure.
+   */
+  @Test
+  public void testCopyFromStream() throws IOException {
+    ByteBuffer buffer = ByteBuffer.allocate(array.length);
+    ByteArrayInputStream bis = new ByteArrayInputStream(array);
+    DataInputStream dis = new DataInputStream(bis);
+
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2);
+    ByteBufferUtils.copyFromStreamToBuffer(buffer, dis,
+        array.length - array.length / 2);
+    for (int i = 0; i < array.length; ++i) {
+      assertEquals(array[i], buffer.get(i));
+    }
+  }
+
+  /**
+   * Test copying from buffer.
+   */
+  @Test
+  public void testCopyFromBuffer() {
+    ByteBuffer srcBuffer = ByteBuffer.allocate(array.length);
+    ByteBuffer dstBuffer = ByteBuffer.allocate(array.length);
+    srcBuffer.put(array);
+
+    ByteBufferUtils.copyFromBufferToBuffer(dstBuffer, srcBuffer,
+        array.length / 2, array.length / 4);
+    for (int i = 0; i < array.length / 4; ++i) {
+      assertEquals(srcBuffer.get(i + array.length / 2),
+          dstBuffer.get(i));
+    }
+  }
+
+  /**
+   * Test 7-bit encoding of integers.
+   * @throws IOException On test failure.
+   */
+  @Test
+  public void testCompressedInt() throws IOException {
+    testCompressedInt(0);
+    testCompressedInt(Integer.MAX_VALUE);
+    testCompressedInt(Integer.MIN_VALUE);
+
+    for (int i = 0; i < 3; i++) {
+      testCompressedInt((128 << i) - 1);
+    }
+
+    for (int i = 0; i < 3; i++) {
+      testCompressedInt((128 << i));
+    }
+  }
+
+  /**
+   * Test how much bytes we need to store integer.
+   */
+  @Test
+  public void testIntFitsIn() {
+    assertEquals(1, ByteBufferUtils.intFitsIn(0));
+    assertEquals(1, ByteBufferUtils.intFitsIn(1));
+    assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8));
+    assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16));
+    assertEquals(4, ByteBufferUtils.intFitsIn(-1));
+    assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE));
+    assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE));
+  }
+
+  /**
+   * Test how much bytes we need to store long.
+   */
+  @Test
+  public void testLongFitsIn() {
+    assertEquals(1, ByteBufferUtils.longFitsIn(0));
+    assertEquals(1, ByteBufferUtils.longFitsIn(1));
+    assertEquals(3, ByteBufferUtils.longFitsIn(1l << 16));
+    assertEquals(5, ByteBufferUtils.longFitsIn(1l << 32));
+    assertEquals(8, ByteBufferUtils.longFitsIn(-1));
+    assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE));
+    assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE));
+  }
+
+  /**
+   * Test if we are comparing equal bytes.
+   */
+  @Test
+  public void testArePartEqual() {
+    byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 };
+    ByteBuffer buffer = ByteBuffer.wrap(array);
+    assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4));
+    assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2));
+    assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3));
+    assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2));
+    assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3));
+  }
+
+  /**
+   * Test serializing int to bytes
+   */
+  @Test
+  public void testPutInt() {
+    testPutInt(0);
+    testPutInt(Integer.MAX_VALUE);
+
+    for (int i = 0; i < 3; i++) {
+      testPutInt((128 << i) - 1);
+    }
+
+    for (int i = 0; i < 3; i++) {
+      testPutInt((128 << i));
+    }
+  }
+
+  // Utility methods invoked from test methods
+
+  private void testCompressedInt(int value) throws IOException {
+    int parsedValue = 0;
+
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    ByteBufferUtils.putCompressedInt(bos, value);
+
+    ByteArrayInputStream bis = new ByteArrayInputStream(
+        bos.toByteArray());
+    parsedValue = ByteBufferUtils.readCompressedInt(bis);
+
+    assertEquals(value, parsedValue);
+  }
+
+  private void testPutInt(int value) {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try {
+      ByteBufferUtils.putInt(baos, value);
+    } catch (IOException e) {
+      throw new RuntimeException("Bug in putIn()", e);
+    }
+
+    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+    DataInputStream dis = new DataInputStream(bais);
+    try {
+      assertEquals(dis.readInt(), value);
+    } catch (IOException e) {
+      throw new RuntimeException("Bug in test!", e);
+    }
+  }
+}



Mime
View raw message