hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1408986 - /hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Date Tue, 13 Nov 2012 22:15:44 GMT
Author: stack
Date: Tue Nov 13 22:15:43 2012
New Revision: 1408986

URL: http://svn.apache.org/viewvc?rev=1408986&view=rev
Log:
HBASE-7156 Add Data Block Encoding and -D opts to Performance Evaluation

Modified:
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=1408986&r1=1408985&r2=1408986&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Tue Nov 13 22:15:43 2012
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.filter.Co
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Hash;
@@ -66,6 +67,7 @@ import org.apache.hadoop.hbase.util.Murm
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
@@ -79,6 +81,8 @@ import org.apache.hadoop.mapreduce.TaskA
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.util.LineReader;
 
 import com.stumbleupon.async.Callback;
@@ -105,7 +109,7 @@ import org.hbase.async.Scanner;
  * <p>If number of clients > 1, we start up a MapReduce job. Each map task
  * runs an individual client. Each client does about 1GB of data.
  */
-public class PerformanceEvaluation {
+public class PerformanceEvaluation extends Configured implements Tool {
   protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
 
   private static final int ROW_LENGTH = 1000;
@@ -121,13 +125,13 @@ public class PerformanceEvaluation {
 
   protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
 
-  volatile Configuration conf;
   private boolean miniCluster = false;
   private boolean nomapred = false;
   private int N = 1;
   private int R = ROWS_PER_GB;
   private byte[] tableName = TABLE_NAME;
   private Compression.Algorithm compression = Compression.Algorithm.NONE;
+  private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
   private boolean flushCommits = true;
   private boolean writeToWAL = true;
   private int presplitRegions = 0;
@@ -157,10 +161,10 @@ public class PerformanceEvaluation {
 
   /**
    * Constructor
-   * @param c Configuration object
+   * @param conf Configuration object
    */
-  public PerformanceEvaluation(final Configuration c) {
-    this.conf = c;
+  public PerformanceEvaluation(final Configuration conf) {
+    super(conf);
 
     addCommandDescriptor(RandomReadTest.class, "randomRead",
         "Run random read test");
@@ -517,6 +521,7 @@ public class PerformanceEvaluation {
     if (TABLE_DESCRIPTOR == null) {
       TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
       HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+      family.setDataBlockEncoding(blockEncoding);
       family.setCompressionType(compression);
       TABLE_DESCRIPTOR.addFamily(family);
     }
@@ -549,7 +554,7 @@ public class PerformanceEvaluation {
    */
   private void runNIsMoreThanOne(final Class<? extends Test> cmd)
   throws IOException, InterruptedException, ClassNotFoundException {
-    checkTable(new HBaseAdmin(conf));
+    checkTable(new HBaseAdmin(getConf()));
     if (this.nomapred) {
       doMultipleClients(cmd);
     } else {
@@ -572,7 +577,7 @@ public class PerformanceEvaluation {
         @Override
         public void run() {
           super.run();
-          PerformanceEvaluation pe = new PerformanceEvaluation(conf);
+          PerformanceEvaluation pe = new PerformanceEvaluation(getConf());
           pe.N = N;
           try {
             long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
@@ -627,10 +632,11 @@ public class PerformanceEvaluation {
    */
   private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
         InterruptedException, ClassNotFoundException {
-    Path inputDir = writeInputFile(this.conf);
-    this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
-    this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
-    Job job = new Job(this.conf);
+    Configuration conf = getConf();
+    Path inputDir = writeInputFile(conf);
+    conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
+    conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
+    Job job = new Job(conf);
     job.setJarByClass(PerformanceEvaluation.class);
     job.setJobName("HBase Performance Evaluation");
 
@@ -1470,7 +1476,7 @@ public class PerformanceEvaluation {
     try {
       Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
           Configuration.class, TestOptions.class, Status.class);
-      t = constructor.newInstance(this.conf, options, status);
+      t = constructor.newInstance(getConf(), options, status);
     } catch (NoSuchMethodException e) {
       throw new IllegalArgumentException("Invalid command class: " +
           cmd.getName() + ".  It does not provide a constructor as described by" +
@@ -1495,7 +1501,7 @@ public class PerformanceEvaluation {
 
     HBaseAdmin admin = null;
     try {
-      admin = new HBaseAdmin(this.conf);
+      admin = new HBaseAdmin(getConf());
       checkTable(admin);
       runOneClient(cmd, 0, this.R, this.R, this.flushCommits, this.writeToWAL,
         status);
@@ -1509,6 +1515,7 @@ public class PerformanceEvaluation {
     MiniHBaseCluster hbaseMiniCluster = null;
     MiniDFSCluster dfsCluster = null;
     MiniZooKeeperCluster zooKeeperCluster = null;
+    Configuration conf = getConf();
     if (this.miniCluster) {
       dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
       zooKeeperCluster = new MiniZooKeeperCluster();
@@ -1523,7 +1530,7 @@ public class PerformanceEvaluation {
       conf.set(HConstants.HBASE_DIR, parentdir.toString());
       fs.mkdirs(parentdir);
       FSUtils.setVersion(fs, parentdir);
-      hbaseMiniCluster = new MiniHBaseCluster(this.conf, N);
+      hbaseMiniCluster = new MiniHBaseCluster(conf, N);
     }
 
     try {
@@ -1554,7 +1561,7 @@ public class PerformanceEvaluation {
     }
     System.err.println("Usage: java " + this.getClass().getName() + " \\");
     System.err.println("  [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
-    System.err.println("  [--compress=TYPE] <command> <nclients>");
+    System.err.println("  [--compress=TYPE] [--blockEncoding=TYPE] [-D<property=value>]*
<command> <nclients>");
     System.err.println();
     System.err.println("Options:");
     System.err.println(" miniCluster     Run the test on an HBaseMiniCluster");
@@ -1567,6 +1574,11 @@ public class PerformanceEvaluation {
     System.err.println(" writeToWAL      Set writeToWAL on puts. Default: True");
     System.err.println(" presplit        Create presplit table. Recommended for accurate
perf analysis (see guide).  Default: disabled");
     System.err.println();
+    System.err.println(" Note: -D properties will be applied to the conf used. ");
+    System.err.println("  For example: ");
+    System.err.println("   -Dmapred.output.compress=true");
+    System.err.println("   -Dmapreduce.task.timeout=60000");
+    System.err.println();
     System.err.println("Command:");
     for (CmdDescriptor command : commands.values()) {
       System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
@@ -1594,7 +1606,7 @@ public class PerformanceEvaluation {
     this.R = this.R * N;
   }
 
-  public int doCommandLine(final String[] args) {
+  public int run(String[] args) throws Exception {
     // Process command-line args. TODO: Better cmd-line processing
     // (but hopefully something not as painful as cli options).
     int errCode = -1;
@@ -1642,6 +1654,12 @@ public class PerformanceEvaluation {
           continue;
         }
 
+        final String blockEncoding = "--blockEncoding=";
+        if (cmd.startsWith(blockEncoding)) {
+          this.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
+          continue;
+        }
+
         final String flushCommits = "--flushCommits=";
         if (cmd.startsWith(flushCommits)) {
           this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
@@ -1686,8 +1704,8 @@ public class PerformanceEvaluation {
   /**
    * @param args
    */
-  public static void main(final String[] args) {
-    Configuration c = HBaseConfiguration.create();
-    System.exit(new PerformanceEvaluation(c).doCommandLine(args));
+  public static void main(final String[] args) throws Exception {
+    int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
+    System.exit(res);
   }
 }



Mime
View raw message