hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [2/2] git commit: HBASE-11691 Backport HBASE-7156 (Add Data Block Encoding and -D opts to Performance Evaluation) to 0.94
Date Thu, 07 Aug 2014 22:24:36 GMT
HBASE-11691 Backport HBASE-7156 (Add Data Block Encoding and -D opts to Performance Evaluation)
to 0.94


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a8142d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a8142d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a8142d1

Branch: refs/heads/0.94
Commit: 9a8142d11f39f37a24486f77ac989a483643a1ba
Parents: 46c2694
Author: Andrew Purtell <apurtell@apache.org>
Authored: Thu Aug 7 15:22:42 2014 -0700
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Thu Aug 7 15:22:42 2014 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/PerformanceEvaluation.java     | 56 +++++++++++++-------
 1 file changed, 37 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a8142d1/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 37ad8e2..cca8210 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -42,6 +42,7 @@ import java.lang.reflect.Constructor;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -82,6 +84,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
 import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * Script used evaluating HBase performance and scalability.  Runs a HBase
@@ -99,7 +103,7 @@ import org.apache.hadoop.util.LineReader;
  * <p>If number of clients > 1, we start up a MapReduce job. Each map task
  * runs an individual client. Each client does about 1GB of data.
  */
-public class PerformanceEvaluation {
+public class PerformanceEvaluation extends Configured implements Tool {
   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
 
   public static final byte[] COMPRESSION = Bytes.toBytes("NONE");
@@ -119,13 +123,13 @@ public class PerformanceEvaluation {
   protected static HTableDescriptor TABLE_DESCRIPTOR;
   protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
 
-  volatile Configuration conf;
   private boolean miniCluster = false;
   private boolean nomapred = false;
   private int N = 1;
   private int R = ROWS_PER_GB;
   private byte[] tableName = TABLE_NAME;
   private Compression.Algorithm compression = Compression.Algorithm.NONE;
+  private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
   private float sampleRate = 1.0f;
   private boolean flushCommits = true;
   private boolean reportLatency = false;
@@ -158,10 +162,10 @@ public class PerformanceEvaluation {
 
   /**
    * Constructor
-   * @param c Configuration object
+   * @param conf Configuration object
    */
-  public PerformanceEvaluation(final Configuration c) {
-    this.conf = c;
+  public PerformanceEvaluation(final Configuration conf) {
+    super(conf);
 
     addCommandDescriptor(RandomReadTest.class, "randomRead",
         "Run random read test");
@@ -517,6 +521,7 @@ public class PerformanceEvaluation {
     if (TABLE_DESCRIPTOR == null) {
       TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
       HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+      family.setDataBlockEncoding(blockEncoding);
       family.setCompressionType(compression);
       TABLE_DESCRIPTOR.addFamily(family);
     }
@@ -549,7 +554,7 @@ public class PerformanceEvaluation {
    */
   private void runNIsMoreThanOne(final Class<? extends Test> cmd)
   throws IOException, InterruptedException, ClassNotFoundException {
-    checkTable(new HBaseAdmin(conf));
+    checkTable(new HBaseAdmin(getConf()));
     if (this.nomapred) {
       doMultipleClients(cmd);
     } else {
@@ -570,7 +575,7 @@ public class PerformanceEvaluation {
         @Override
         public void run() {
           super.run();
-          PerformanceEvaluation pe = new PerformanceEvaluation(conf);
+          PerformanceEvaluation pe = new PerformanceEvaluation(getConf());
           int index = Integer.parseInt(getName());
           try {
             long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
@@ -612,10 +617,11 @@ public class PerformanceEvaluation {
    */
   private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
         InterruptedException, ClassNotFoundException {
-    Path inputDir = writeInputFile(this.conf);
-    this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
-    this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
-    Job job = new Job(this.conf);
+    Configuration conf = getConf();
+    Path inputDir = writeInputFile(conf);
+    conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
+    conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
+    Job job = new Job(conf);
     job.setJarByClass(PerformanceEvaluation.class);
     job.setJobName("HBase Performance Evaluation");
 
@@ -1229,7 +1235,7 @@ public class PerformanceEvaluation {
     try {
       Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
           Configuration.class, TestOptions.class, Status.class);
-      t = constructor.newInstance(this.conf, options, status);
+      t = constructor.newInstance(getConf(), options, status);
     } catch (NoSuchMethodException e) {
       throw new IllegalArgumentException("Invalid command class: " +
           cmd.getName() + ".  It does not provide a constructor as described by" +
@@ -1255,7 +1261,7 @@ public class PerformanceEvaluation {
 
     HBaseAdmin admin = null;
     try {
-      admin = new HBaseAdmin(this.conf);
+      admin = new HBaseAdmin(getConf());
       checkTable(admin);
       runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
         this.writeToWAL, this.writeToWAL, status);
@@ -1271,6 +1277,7 @@ public class PerformanceEvaluation {
     MiniHBaseCluster hbaseMiniCluster = null;
     MiniDFSCluster dfsCluster = null;
     MiniZooKeeperCluster zooKeeperCluster = null;
+    Configuration conf = getConf();
     if (this.miniCluster) {
       dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
       zooKeeperCluster = new MiniZooKeeperCluster();
@@ -1285,7 +1292,7 @@ public class PerformanceEvaluation {
       conf.set(HConstants.HBASE_DIR, parentdir.toString());
       fs.mkdirs(parentdir);
       FSUtils.setVersion(fs, parentdir);
-      hbaseMiniCluster = new MiniHBaseCluster(this.conf, N);
+      hbaseMiniCluster = new MiniHBaseCluster(conf, N);
     }
 
     try {
@@ -1316,7 +1323,7 @@ public class PerformanceEvaluation {
     }
     System.err.println("Usage: java " + this.getClass().getName() + " \\");
     System.err.println("  [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
-    System.err.println("  [--compress=TYPE] <command> <nclients>");
+    System.err.println("  [--compress=TYPE] [--blockEncoding=TYPE] [-D<property=value>]*
<command> <nclients>");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" miniCluster     Run the test on an HBaseMiniCluster");
@@ -1335,6 +1342,11 @@ public class PerformanceEvaluation {
     System.err.println(" latency         Set to report operation latencies. " +
       "Currently only supported by randomRead test. Default: False");
     System.err.println();
+    System.err.println(" Note: -D properties will be applied to the conf used. ");
+    System.err.println("  For example: ");
+    System.err.println("   -Dmapred.output.compress=true");
+    System.err.println("   -Dmapreduce.task.timeout=60000");
+    System.err.println();
     System.err.println("Command:");
     for (CmdDescriptor command : commands.values()) {
       System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
@@ -1362,7 +1374,7 @@ public class PerformanceEvaluation {
     this.R = this.R * N;
   }
 
-  public int doCommandLine(final String[] args) {
+  public int run(String[] args) throws Exception {
     // Process command-line args. TODO: Better cmd-line processing
     // (but hopefully something not as painful as cli options).
     int errCode = -1;
@@ -1416,6 +1428,12 @@ public class PerformanceEvaluation {
           continue;
         }
 
+        final String blockEncoding = "--blockEncoding=";
+        if (cmd.startsWith(blockEncoding)) {
+          this.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
+          continue;
+        }
+
         final String flushCommits = "--flushCommits=";
         if (cmd.startsWith(flushCommits)) {
           this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
@@ -1463,8 +1481,8 @@ public class PerformanceEvaluation {
     return descriptor != null ? descriptor.getCmdClass() : null;
   }
 
-  public static void main(final String[] args) {
-    Configuration c = HBaseConfiguration.create();
-    System.exit(new PerformanceEvaluation(c).doCommandLine(args));
+  public static void main(final String[] args) throws Exception {
+    int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
+    System.exit(res);
   }
 }


Mime
View raw message