hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1480277 - in /hbase/trunk: hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java
Date Wed, 08 May 2013 13:54:26 GMT
Author: tedyu
Date: Wed May  8 13:54:26 2013
New Revision: 1480277

URL: http://svn.apache.org/r1480277
Log:
HBASE-8306 Enhance TestJoinedScanners with ability to specify various parameters (Ted Yu)


Modified:
    hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java

Modified: hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java?rev=1480277&r1=1480276&r2=1480277&view=diff
==============================================================================
--- hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java
(original)
+++ hbase/trunk/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestLazyCfLoading.java
Wed May  8 13:54:26 2013
@@ -18,9 +18,7 @@
 
 package org.apache.hadoop.hbase;
 
-import java.io.IOException;
 import java.security.InvalidParameterException;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -38,17 +36,15 @@ import org.apache.hadoop.hbase.client.Sc
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.LoadTestTool;
 import org.apache.hadoop.hbase.util.MultiThreadedWriter;
 import org.apache.hadoop.hbase.util.RegionSplitter;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
 import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -61,6 +57,7 @@ import org.junit.experimental.categories
 public class IntegrationTestLazyCfLoading {
   private static final String TABLE_NAME = IntegrationTestLazyCfLoading.class.getSimpleName();
   private static final String TIMEOUT_KEY = "hbase.%s.timeout";
+  private static final String ENCODING_KEY = "hbase.%s.datablock.encoding";
 
   /** A soft test timeout; duration of the test, as such, depends on number of keys to put.
*/
   private static final int DEFAULT_TIMEOUT_MINUTES = 10;
@@ -183,9 +180,14 @@ public class IntegrationTestLazyCfLoadin
   private void createTable() throws Exception {
     deleteTable();
     LOG.info("Creating table");
+    Configuration conf = util.getConfiguration();
+    String encodingKey = String.format(ENCODING_KEY, this.getClass().getSimpleName());
+    DataBlockEncoding blockEncoding = DataBlockEncoding.valueOf(conf.get(encodingKey, "FAST_DIFF"));
     HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes(TABLE_NAME));
     for (byte[] cf : dataGen.getColumnFamilies()) {
-      htd.addFamily(new HColumnDescriptor(cf));
+      HColumnDescriptor hcd = new HColumnDescriptor(cf);
+      hcd.setDataBlockEncoding(blockEncoding);
+      htd.addFamily(hcd);
     }
     int serverCount = util.getHBaseClusterInterface().getClusterStatus().getServersSize();
     byte[][] splits = new RegionSplitter.HexStringSplit().split(serverCount * REGIONS_PER_SERVER);

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java?rev=1480277&r1=1480276&r2=1480277&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestJoinedScanners.java
Wed May  8 13:54:26 2013
@@ -21,7 +21,14 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -40,12 +47,12 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 
-
 /**
  * Test performance improvement of joined scanners optimization:
  * https://issues.apache.org/jira/browse/HBASE-5416
@@ -63,6 +70,10 @@ public class TestJoinedScanners {
   private static final byte[] flag_yes = Bytes.toBytes("Y");
   private static final byte[] flag_no  = Bytes.toBytes("N");
 
+  private static DataBlockEncoding blockEncoding = DataBlockEncoding.FAST_DIFF;
+  private static int selectionRatio = 30;
+  private static int valueWidth = 128 * 1024;
+
   @Test
   public void testJoinedScanners() throws Exception {
     String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
@@ -80,28 +91,33 @@ public class TestJoinedScanners {
       cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
       byte [][] families = {cf_essential, cf_joined};
 
-      HTable ht = htu.createTable(
-        Bytes.toBytes(this.getClass().getSimpleName()), families);
+      byte[] tableName = Bytes.toBytes(this.getClass().getSimpleName());
+      HTableDescriptor desc = new HTableDescriptor(tableName);
+      for(byte[] family : families) {
+        HColumnDescriptor hcd = new HColumnDescriptor(family);
+        hcd.setDataBlockEncoding(blockEncoding);
+        desc.addFamily(hcd);
+      }
+      htu.getHBaseAdmin().createTable(desc);
+      HTable ht = new HTable(htu.getConfiguration(), tableName);
 
       long rows_to_insert = 1000;
       int insert_batch = 20;
-      int flag_percent = 1;
-      int large_bytes = 128 * 1024;
       long time = System.nanoTime();
+      Random rand = new Random(time);
 
       LOG.info("Make " + Long.toString(rows_to_insert) + " rows, total size = "
-        + Float.toString(rows_to_insert * large_bytes / 1024 / 1024) + " MB");
+        + Float.toString(rows_to_insert * valueWidth / 1024 / 1024) + " MB");
 
-      byte [] val_large = new byte[large_bytes];
+      byte [] val_large = new byte[valueWidth];
 
       List<Put> puts = new ArrayList<Put>();
 
       for (long i = 0; i < rows_to_insert; i++) {
         Put put = new Put(Bytes.toBytes(Long.toString (i)));
-        if (i % 100 <= flag_percent) {
+        if (rand.nextInt(100) <= selectionRatio) {
           put.add(cf_essential, col_name, flag_yes);
-        }
-        else {
+        } else {
           put.add(cf_essential, col_name, flag_no);
         }
         put.add(cf_joined, col_name, val_large);
@@ -120,7 +136,7 @@ public class TestJoinedScanners {
         + Double.toString((System.nanoTime() - time) / 1000000000.0) + " seconds");
 
       boolean slow = true;
-      for (int i = 0; i < 20; ++i) {
+      for (int i = 0; i < 10; ++i) {
         runScanner(ht, slow);
         slow = !slow;
       }
@@ -163,7 +179,9 @@ public class TestJoinedScanners {
       throws IOException {
     HTableDescriptor htd = new HTableDescriptor(tableName);
     for(byte [] family : families) {
-      htd.addFamily(new HColumnDescriptor(family));
+      HColumnDescriptor hcd = new HColumnDescriptor(family);
+      hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
+      htd.addFamily(hcd);
     }
     HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false);
     Path path = new Path(DIR + callingMethod);
@@ -175,4 +193,48 @@ public class TestJoinedScanners {
     }
     return HRegion.createHRegion(info, path, conf, htd);
   }
-}
\ No newline at end of file
+
+  private static Options options = new Options();
+
+  /**
+   * Command line interface:
+   * @param args
+   * @throws IOException if there is a bug while reading from disk
+   */
+  public static void main(final String[] args) throws Exception {
+    Option encodingOption = new Option("e", "blockEncoding", true,
+      "Data block encoding; Default: FAST_DIFF");
+    encodingOption.setRequired(false);
+    options.addOption(encodingOption);
+    
+    Option ratioOption = new Option("r", "selectionRatio", true,
+      "Ratio of selected rows using essential column family");
+    ratioOption.setRequired(false);
+    options.addOption(ratioOption);
+    
+    Option widthOption = new Option("w", "valueWidth", true,
+      "Width of value for non-essential column family");
+    widthOption.setRequired(false);
+    options.addOption(widthOption);
+    
+    CommandLineParser parser = new GnuParser();
+    CommandLine cmd = parser.parse(options, args);
+    if (args.length < 1) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp("TestJoinedScanners", options, true);
+    }
+    
+    if (cmd.hasOption("e")) {
+      blockEncoding = DataBlockEncoding.valueOf(cmd.getOptionValue("e"));
+    }
+    if (cmd.hasOption("r")) {
+      selectionRatio = Integer.parseInt(cmd.getOptionValue("r"));
+    }
+    if (cmd.hasOption("w")) {
+      valueWidth = Integer.parseInt(cmd.getOptionValue("w"));
+    }
+    // run the test
+    TestJoinedScanners test = new TestJoinedScanners();
+    test.testJoinedScanners();
+  }
+}



Mime
View raw message