hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [01/41] hbase git commit: HBASE-18640 Move mapreduce out of hbase-server into separate module.
Date Sat, 26 Aug 2017 08:55:47 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-2 95bc46452 -> 59d034106


http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
deleted file mode 100644
index ad832e3..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java
+++ /dev/null
@@ -1,968 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.lang.reflect.Constructor;
-import java.net.InetAddress;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Properties;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hadoop.hbase.io.crypto.Cipher;
-import org.apache.hadoop.hbase.io.crypto.Encryption;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.security.EncryptionUtil;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.security.access.AccessControlClient;
-import org.apache.hadoop.hbase.security.access.Permission;
-import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
-import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * A command-line utility that reads, writes, and verifies data. Unlike
- * {@link org.apache.hadoop.hbase.PerformanceEvaluation}, this tool validates the data written,
- * and supports simultaneously writing and reading the same set of keys.
- */
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
-public class LoadTestTool extends AbstractHBaseTool {
-
-  private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
-  private static final String COLON = ":";
-
-  /** Table name for the test */
-  private TableName tableName;
-
-  /** Column families for the test */
-  private byte[][] families;
-
-  /** Table name to use of not overridden on the command line */
-  protected static final String DEFAULT_TABLE_NAME = "cluster_test";
-
-  /** Column family used by the test */
-  public static byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("test_cf");
-
-  /** Column families used by the test */
-  public static final byte[][] DEFAULT_COLUMN_FAMILIES = { DEFAULT_COLUMN_FAMILY };
-
-  /** The default data size if not specified */
-  protected static final int DEFAULT_DATA_SIZE = 64;
-
-  /** The number of reader/writer threads if not specified */
-  protected static final int DEFAULT_NUM_THREADS = 20;
-
-  /** Usage string for the load option */
-  protected static final String OPT_USAGE_LOAD =
-      "<avg_cols_per_key>:<avg_data_size>" +
-      "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
-
-  /** Usage string for the read option */
-  protected static final String OPT_USAGE_READ =
-      "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
-
-  /** Usage string for the update option */
-  protected static final String OPT_USAGE_UPDATE =
-      "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS
-      + ">][:<#whether to ignore nonce collisions=0>]";
-
-  protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " +
-      Arrays.toString(BloomType.values());
-
-  protected static final String OPT_USAGE_COMPRESSION = "Compression type, " +
-      "one of " + Arrays.toString(Compression.Algorithm.values());
-
-  public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
-    "Encoding algorithm (e.g. prefix "
-        + "compression) to use for data blocks in the test column family, "
-        + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
-
-  public static final String OPT_BLOOM = "bloom";
-  public static final String OPT_COMPRESSION = "compression";
-  public static final String OPT_DEFERRED_LOG_FLUSH = "deferredlogflush";
-  public static final String OPT_DEFERRED_LOG_FLUSH_USAGE = "Enable deferred log flush.";
-
-  public static final String OPT_DATA_BLOCK_ENCODING =
-      HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT);
-
-  public static final String OPT_INMEMORY = "in_memory";
-  public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " +
-      "inmemory as far as possible.  Not guaranteed that reads are always served from inmemory";
-
-  public static final String OPT_GENERATOR = "generator";
-  public static final String OPT_GENERATOR_USAGE = "The class which generates load for the tool."
-      + " Any args for this class can be passed as colon separated after class name";
-
-  public static final String OPT_WRITER = "writer";
-  public static final String OPT_WRITER_USAGE = "The class for executing the write requests";
-
-  public static final String OPT_UPDATER = "updater";
-  public static final String OPT_UPDATER_USAGE = "The class for executing the update requests";
-
-  public static final String OPT_READER = "reader";
-  public static final String OPT_READER_USAGE = "The class for executing the read requests";
-
-  protected static final String OPT_KEY_WINDOW = "key_window";
-  protected static final String OPT_WRITE = "write";
-  protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
-  public static final String OPT_MULTIPUT = "multiput";
-  public static final String OPT_MULTIGET = "multiget_batchsize";
-  protected static final String OPT_NUM_KEYS = "num_keys";
-  protected static final String OPT_READ = "read";
-  protected static final String OPT_START_KEY = "start_key";
-  public static final String OPT_TABLE_NAME = "tn";
-  public static final String OPT_COLUMN_FAMILIES = "families";
-  protected static final String OPT_ZK_QUORUM = "zk";
-  protected static final String OPT_ZK_PARENT_NODE = "zk_root";
-  protected static final String OPT_SKIP_INIT = "skip_init";
-  protected static final String OPT_INIT_ONLY = "init_only";
-  protected static final String NUM_TABLES = "num_tables";
-  protected static final String OPT_REGIONS_PER_SERVER = "regions_per_server";
-  protected static final String OPT_BATCHUPDATE = "batchupdate";
-  protected static final String OPT_UPDATE = "update";
-
-  public static final String OPT_ENCRYPTION = "encryption";
-  protected static final String OPT_ENCRYPTION_USAGE =
-    "Enables transparent encryption on the test table, one of " +
-    Arrays.toString(Encryption.getSupportedCiphers());
-
-  public static final String OPT_NUM_REGIONS_PER_SERVER = "num_regions_per_server";
-  protected static final String OPT_NUM_REGIONS_PER_SERVER_USAGE
-    = "Desired number of regions per region server. Defaults to 5.";
-  public static int DEFAULT_NUM_REGIONS_PER_SERVER = 5;
-
-  public static final String OPT_REGION_REPLICATION = "region_replication";
-  protected static final String OPT_REGION_REPLICATION_USAGE =
-      "Desired number of replicas per region";
-
-  public static final String OPT_REGION_REPLICA_ID = "region_replica_id";
-  protected static final String OPT_REGION_REPLICA_ID_USAGE =
-      "Region replica id to do the reads from";
-
-  public static final String OPT_MOB_THRESHOLD = "mob_threshold";
-  protected static final String OPT_MOB_THRESHOLD_USAGE =
-      "Desired cell size to exceed in bytes that will use the MOB write path";
-
-  protected static final long DEFAULT_START_KEY = 0;
-
-  /** This will be removed as we factor out the dependency on command line */
-  protected CommandLine cmd;
-
-  protected MultiThreadedWriter writerThreads = null;
-  protected MultiThreadedReader readerThreads = null;
-  protected MultiThreadedUpdater updaterThreads = null;
-
-  protected long startKey, endKey;
-
-  protected boolean isWrite, isRead, isUpdate;
-  protected boolean deferredLogFlush;
-
-  // Column family options
-  protected DataBlockEncoding dataBlockEncodingAlgo;
-  protected Compression.Algorithm compressAlgo;
-  protected BloomType bloomType;
-  private boolean inMemoryCF;
-
-  private User userOwner;
-  // Writer options
-  protected int numWriterThreads = DEFAULT_NUM_THREADS;
-  protected int minColsPerKey, maxColsPerKey;
-  protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
-  protected boolean isMultiPut;
-
-  // Updater options
-  protected int numUpdaterThreads = DEFAULT_NUM_THREADS;
-  protected int updatePercent;
-  protected boolean ignoreConflicts = false;
-  protected boolean isBatchUpdate;
-
-  // Reader options
-  private int numReaderThreads = DEFAULT_NUM_THREADS;
-  private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
-  private int multiGetBatchSize = MultiThreadedReader.DEFAULT_BATCH_SIZE;
-  private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
-  private int verifyPercent;
-
-  private int numTables = 1;
-
-  private String superUser;
-
-  private String userNames;
-  //This file is used to read authentication information in secure clusters.
-  private String authnFileName;
-
-  private int numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER;
-  private int regionReplication = -1; // not set
-  private int regionReplicaId = -1; // not set
-
-  private int mobThreshold = -1; // not set
-
-  // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
-  //       console tool itself should only be used from console.
-  protected boolean isSkipInit = false;
-  protected boolean isInitOnly = false;
-
-  protected Cipher cipher = null;
-
-  protected String[] splitColonSeparated(String option,
-      int minNumCols, int maxNumCols) {
-    String optVal = cmd.getOptionValue(option);
-    String[] cols = optVal.split(COLON);
-    if (cols.length < minNumCols || cols.length > maxNumCols) {
-      throw new IllegalArgumentException("Expected at least "
-          + minNumCols + " columns but no more than " + maxNumCols +
-          " in the colon-separated value '" + optVal + "' of the " +
-          "-" + option + " option");
-    }
-    return cols;
-  }
-
-  protected int getNumThreads(String numThreadsStr) {
-    return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
-  }
-
-  public byte[][] getColumnFamilies() {
-    return families;
-  }
-
-  /**
-   * Apply column family options such as Bloom filters, compression, and data
-   * block encoding.
-   */
-  protected void applyColumnFamilyOptions(TableName tableName,
-      byte[][] columnFamilies) throws IOException {
-    try (Connection conn = ConnectionFactory.createConnection(conf);
-        Admin admin = conn.getAdmin()) {
-      TableDescriptor tableDesc = admin.getTableDescriptor(tableName);
-      LOG.info("Disabling table " + tableName);
-      admin.disableTable(tableName);
-      for (byte[] cf : columnFamilies) {
-        HColumnDescriptor columnDesc = (HColumnDescriptor) tableDesc.getColumnFamily(cf);
-        boolean isNewCf = columnDesc == null;
-        if (isNewCf) {
-          columnDesc = new HColumnDescriptor(cf);
-        }
-        if (bloomType != null) {
-          columnDesc.setBloomFilterType(bloomType);
-        }
-        if (compressAlgo != null) {
-          columnDesc.setCompressionType(compressAlgo);
-        }
-        if (dataBlockEncodingAlgo != null) {
-          columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
-        }
-        if (inMemoryCF) {
-          columnDesc.setInMemory(inMemoryCF);
-        }
-        if (cipher != null) {
-          byte[] keyBytes = new byte[cipher.getKeyLength()];
-          new SecureRandom().nextBytes(keyBytes);
-          columnDesc.setEncryptionType(cipher.getName());
-          columnDesc.setEncryptionKey(
-              EncryptionUtil.wrapKey(conf,
-                  User.getCurrent().getShortName(),
-                  new SecretKeySpec(keyBytes,
-                      cipher.getName())));
-        }
-        if (mobThreshold >= 0) {
-          columnDesc.setMobEnabled(true);
-          columnDesc.setMobThreshold(mobThreshold);
-        }
-
-        if (isNewCf) {
-          admin.addColumnFamily(tableName, columnDesc);
-        } else {
-          admin.modifyColumnFamily(tableName, columnDesc);
-        }
-      }
-      LOG.info("Enabling table " + tableName);
-      admin.enableTable(tableName);
-    }
-  }
-
-  @Override
-  protected void addOptions() {
-    addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " +
-        "without port numbers");
-    addOptWithArg(OPT_ZK_PARENT_NODE, "name of parent znode in zookeeper");
-    addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
-    addOptWithArg(OPT_COLUMN_FAMILIES, "The name of the column families to use separated by comma");
-    addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
-    addOptWithArg(OPT_READ, OPT_USAGE_READ);
-    addOptWithArg(OPT_UPDATE, OPT_USAGE_UPDATE);
-    addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
-    addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
-    addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
-    addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
-    addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " +
-        "to tolerate before terminating all reader threads. The default is " +
-        MultiThreadedReader.DEFAULT_MAX_ERRORS + ".");
-    addOptWithArg(OPT_MULTIGET, "Whether to use multi-gets as opposed to " +
-        "separate gets for every column in a row");
-    addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " +
-        "reads and writes for concurrent write/read workload. The default " +
-        "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + ".");
-
-    addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " +
-        "separate puts for every column in a row");
-    addOptNoArg(OPT_BATCHUPDATE, "Whether to use batch as opposed to " +
-        "separate updates for every column in a row");
-    addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
-    addOptWithArg(OPT_GENERATOR, OPT_GENERATOR_USAGE);
-    addOptWithArg(OPT_WRITER, OPT_WRITER_USAGE);
-    addOptWithArg(OPT_UPDATER, OPT_UPDATER_USAGE);
-    addOptWithArg(OPT_READER, OPT_READER_USAGE);
-
-    addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
-    addOptWithArg(OPT_START_KEY, "The first key to read/write " +
-        "(a 0-based index). The default value is " +
-        DEFAULT_START_KEY + ".");
-    addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table "
-        + "already exists");
-
-    addOptWithArg(NUM_TABLES,
-      "A positive integer number. When a number n is speicfied, load test "
-          + "tool  will load n table parallely. -tn parameter value becomes "
-          + "table name prefix. Each table name is in format <tn>_1...<tn>_n");
-
-    addOptWithArg(OPT_REGIONS_PER_SERVER,
-      "A positive integer number. When a number n is specified, load test "
-          + "tool will create the test table with n regions per server");
-
-    addOptWithArg(OPT_ENCRYPTION, OPT_ENCRYPTION_USAGE);
-    addOptNoArg(OPT_DEFERRED_LOG_FLUSH, OPT_DEFERRED_LOG_FLUSH_USAGE);
-    addOptWithArg(OPT_NUM_REGIONS_PER_SERVER, OPT_NUM_REGIONS_PER_SERVER_USAGE);
-    addOptWithArg(OPT_REGION_REPLICATION, OPT_REGION_REPLICATION_USAGE);
-    addOptWithArg(OPT_REGION_REPLICA_ID, OPT_REGION_REPLICA_ID_USAGE);
-    addOptWithArg(OPT_MOB_THRESHOLD, OPT_MOB_THRESHOLD_USAGE);
-  }
-
-  @Override
-  protected void processOptions(CommandLine cmd) {
-    this.cmd = cmd;
-
-    tableName = TableName.valueOf(cmd.getOptionValue(OPT_TABLE_NAME,
-        DEFAULT_TABLE_NAME));
-
-    if (cmd.hasOption(OPT_COLUMN_FAMILIES)) {
-      String[] list = cmd.getOptionValue(OPT_COLUMN_FAMILIES).split(",");
-      families = new byte[list.length][];
-      for (int i = 0; i < list.length; i++) {
-        families[i] = Bytes.toBytes(list[i]);
-      }
-    } else {
-      families = DEFAULT_COLUMN_FAMILIES;
-    }
-
-    isWrite = cmd.hasOption(OPT_WRITE);
-    isRead = cmd.hasOption(OPT_READ);
-    isUpdate = cmd.hasOption(OPT_UPDATE);
-    isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
-    deferredLogFlush = cmd.hasOption(OPT_DEFERRED_LOG_FLUSH);
-
-    if (!isWrite && !isRead && !isUpdate && !isInitOnly) {
-      throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " +
-        "-" + OPT_UPDATE + " or -" + OPT_READ + " has to be specified");
-    }
-
-    if (isInitOnly && (isRead || isWrite || isUpdate)) {
-      throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with"
-          + " either -" + OPT_WRITE + " or -" + OPT_UPDATE + " or -" + OPT_READ);
-    }
-
-    if (!isInitOnly) {
-      if (!cmd.hasOption(OPT_NUM_KEYS)) {
-        throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in "
-            + "read or write mode");
-      }
-      startKey = parseLong(cmd.getOptionValue(OPT_START_KEY,
-          String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE);
-      long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1,
-          Long.MAX_VALUE - startKey);
-      endKey = startKey + numKeys;
-      isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
-      System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
-    }
-
-    parseColumnFamilyOptions(cmd);
-
-    if (isWrite) {
-      String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
-
-      int colIndex = 0;
-      minColsPerKey = 1;
-      maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
-      int avgColDataSize =
-          parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
-      minColDataSize = avgColDataSize / 2;
-      maxColDataSize = avgColDataSize * 3 / 2;
-
-      if (colIndex < writeOpts.length) {
-        numWriterThreads = getNumThreads(writeOpts[colIndex++]);
-      }
-
-      isMultiPut = cmd.hasOption(OPT_MULTIPUT);
-
-      mobThreshold = -1;
-      if (cmd.hasOption(OPT_MOB_THRESHOLD)) {
-        mobThreshold = Integer.parseInt(cmd.getOptionValue(OPT_MOB_THRESHOLD));
-      }
-
-      System.out.println("Multi-puts: " + isMultiPut);
-      System.out.println("Columns per key: " + minColsPerKey + ".."
-          + maxColsPerKey);
-      System.out.println("Data size per column: " + minColDataSize + ".."
-          + maxColDataSize);
-    }
-
-    if (isUpdate) {
-      String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3);
-      int colIndex = 0;
-      updatePercent = parseInt(mutateOpts[colIndex++], 0, 100);
-      if (colIndex < mutateOpts.length) {
-        numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]);
-      }
-      if (colIndex < mutateOpts.length) {
-        ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1;
-      }
-
-      isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE);
-
-      System.out.println("Batch updates: " + isBatchUpdate);
-      System.out.println("Percent of keys to update: " + updatePercent);
-      System.out.println("Updater threads: " + numUpdaterThreads);
-      System.out.println("Ignore nonce conflicts: " + ignoreConflicts);
-    }
-
-    if (isRead) {
-      String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
-      int colIndex = 0;
-      verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
-      if (colIndex < readOpts.length) {
-        numReaderThreads = getNumThreads(readOpts[colIndex++]);
-      }
-
-      if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
-        maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS),
-            0, Integer.MAX_VALUE);
-      }
-
-      if (cmd.hasOption(OPT_KEY_WINDOW)) {
-        keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW),
-            0, Integer.MAX_VALUE);
-      }
-
-      if (cmd.hasOption(OPT_MULTIGET)) {
-        multiGetBatchSize = parseInt(cmd.getOptionValue(OPT_MULTIGET),
-            0, Integer.MAX_VALUE);
-      }
-
-      System.out.println("Multi-gets (value of 1 means no multigets): " + multiGetBatchSize);
-      System.out.println("Percent of keys to verify: " + verifyPercent);
-      System.out.println("Reader threads: " + numReaderThreads);
-    }
-
-    numTables = 1;
-    if (cmd.hasOption(NUM_TABLES)) {
-      numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
-    }
-
-    numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER;
-    if (cmd.hasOption(OPT_NUM_REGIONS_PER_SERVER)) {
-      numRegionsPerServer = Integer.parseInt(cmd.getOptionValue(OPT_NUM_REGIONS_PER_SERVER));
-    }
-
-    regionReplication = 1;
-    if (cmd.hasOption(OPT_REGION_REPLICATION)) {
-      regionReplication = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICATION));
-    }
-
-    regionReplicaId = -1;
-    if (cmd.hasOption(OPT_REGION_REPLICA_ID)) {
-      regionReplicaId = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICA_ID));
-    }
-  }
-
-  private void parseColumnFamilyOptions(CommandLine cmd) {
-    String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
-    dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
-        DataBlockEncoding.valueOf(dataBlockEncodingStr);
-
-    String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
-    compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
-        Compression.Algorithm.valueOf(compressStr);
-
-    String bloomStr = cmd.getOptionValue(OPT_BLOOM);
-    bloomType = bloomStr == null ? BloomType.ROW :
-        BloomType.valueOf(bloomStr);
-
-    inMemoryCF = cmd.hasOption(OPT_INMEMORY);
-    if (cmd.hasOption(OPT_ENCRYPTION)) {
-      cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION));
-    }
-
-  }
-
-  public void initTestTable() throws IOException {
-    Durability durability = Durability.USE_DEFAULT;
-    if (deferredLogFlush) {
-      durability = Durability.ASYNC_WAL;
-    }
-
-    HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName,
-      getColumnFamilies(), compressAlgo, dataBlockEncodingAlgo, numRegionsPerServer,
-        regionReplication, durability);
-    applyColumnFamilyOptions(tableName, getColumnFamilies());
-  }
-
-  @Override
-  protected int doWork() throws IOException {
-    if (numTables > 1) {
-      return parallelLoadTables();
-    } else {
-      return loadTable();
-    }
-  }
-
-  protected int loadTable() throws IOException {
-    if (cmd.hasOption(OPT_ZK_QUORUM)) {
-      conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
-    }
-    if (cmd.hasOption(OPT_ZK_PARENT_NODE)) {
-      conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, cmd.getOptionValue(OPT_ZK_PARENT_NODE));
-    }
-
-    if (isInitOnly) {
-      LOG.info("Initializing only; no reads or writes");
-      initTestTable();
-      return 0;
-    }
-
-    if (!isSkipInit) {
-      initTestTable();
-    }
-    LoadTestDataGenerator dataGen = null;
-    if (cmd.hasOption(OPT_GENERATOR)) {
-      String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON);
-      dataGen = getLoadGeneratorInstance(clazzAndArgs[0]);
-      String[] args;
-      if (dataGen instanceof LoadTestDataGeneratorWithACL) {
-        LOG.info("Using LoadTestDataGeneratorWithACL");
-        if (User.isHBaseSecurityEnabled(conf)) {
-          LOG.info("Security is enabled");
-          authnFileName = clazzAndArgs[1];
-          superUser = clazzAndArgs[2];
-          userNames = clazzAndArgs[3];
-          args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length);
-          Properties authConfig = new Properties();
-          authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName));
-          try {
-            addAuthInfoToConf(authConfig, conf, superUser, userNames);
-          } catch (IOException exp) {
-            LOG.error(exp);
-            return EXIT_FAILURE;
-          }
-          userOwner = User.create(loginAndReturnUGI(conf, superUser));
-        } else {
-          superUser = clazzAndArgs[1];
-          userNames = clazzAndArgs[2];
-          args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length);
-          userOwner = User.createUserForTesting(conf, superUser, new String[0]);
-        }
-      } else {
-        args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1,
-            clazzAndArgs.length);
-      }
-      dataGen.initialize(args);
-    } else {
-      // Default DataGenerator is MultiThreadedAction.DefaultDataGenerator
-      dataGen = new MultiThreadedAction.DefaultDataGenerator(minColDataSize, maxColDataSize,
-          minColsPerKey, maxColsPerKey, families);
-    }
-
-    if (userOwner != null) {
-      LOG.info("Granting permissions for user " + userOwner.getShortName());
-      Permission.Action[] actions = {
-        Permission.Action.ADMIN, Permission.Action.CREATE,
-        Permission.Action.READ, Permission.Action.WRITE };
-      try {
-        AccessControlClient.grant(ConnectionFactory.createConnection(conf),
-            tableName, userOwner.getShortName(), null, null, actions);
-      } catch (Throwable e) {
-        LOG.fatal("Error in granting permission for the user " + userOwner.getShortName(), e);
-        return EXIT_FAILURE;
-      }
-    }
-
-    if (userNames != null) {
-      // This will be comma separated list of expressions.
-      String users[] = userNames.split(",");
-      User user = null;
-      for (String userStr : users) {
-        if (User.isHBaseSecurityEnabled(conf)) {
-          user = User.create(loginAndReturnUGI(conf, userStr));
-        } else {
-          user = User.createUserForTesting(conf, userStr, new String[0]);
-        }
-      }
-    }
-
-    if (isWrite) {
-      if (userOwner != null) {
-        writerThreads = new MultiThreadedWriterWithACL(dataGen, conf, tableName, userOwner);
-      } else {
-        String writerClass = null;
-        if (cmd.hasOption(OPT_WRITER)) {
-          writerClass = cmd.getOptionValue(OPT_WRITER);
-        } else {
-          writerClass = MultiThreadedWriter.class.getCanonicalName();
-        }
-
-        writerThreads = getMultiThreadedWriterInstance(writerClass, dataGen);
-      }
-      writerThreads.setMultiPut(isMultiPut);
-    }
-
-    if (isUpdate) {
-      if (userOwner != null) {
-        updaterThreads = new MultiThreadedUpdaterWithACL(dataGen, conf, tableName, updatePercent,
-            userOwner, userNames);
-      } else {
-        String updaterClass = null;
-        if (cmd.hasOption(OPT_UPDATER)) {
-          updaterClass = cmd.getOptionValue(OPT_UPDATER);
-        } else {
-          updaterClass = MultiThreadedUpdater.class.getCanonicalName();
-        }
-        updaterThreads = getMultiThreadedUpdaterInstance(updaterClass, dataGen);
-      }
-      updaterThreads.setBatchUpdate(isBatchUpdate);
-      updaterThreads.setIgnoreNonceConflicts(ignoreConflicts);
-    }
-
-    if (isRead) {
-      if (userOwner != null) {
-        readerThreads = new MultiThreadedReaderWithACL(dataGen, conf, tableName, verifyPercent,
-            userNames);
-      } else {
-        String readerClass = null;
-        if (cmd.hasOption(OPT_READER)) {
-          readerClass = cmd.getOptionValue(OPT_READER);
-        } else {
-          readerClass = MultiThreadedReader.class.getCanonicalName();
-        }
-        readerThreads = getMultiThreadedReaderInstance(readerClass, dataGen);
-      }
-      readerThreads.setMaxErrors(maxReadErrors);
-      readerThreads.setKeyWindow(keyWindow);
-      readerThreads.setMultiGetBatchSize(multiGetBatchSize);
-      readerThreads.setRegionReplicaId(regionReplicaId);
-    }
-
-    if (isUpdate && isWrite) {
-      LOG.info("Concurrent write/update workload: making updaters aware of the " +
-        "write point");
-      updaterThreads.linkToWriter(writerThreads);
-    }
-
-    if (isRead && (isUpdate || isWrite)) {
-      LOG.info("Concurrent write/read workload: making readers aware of the " +
-        "write point");
-      readerThreads.linkToWriter(isUpdate ? updaterThreads : writerThreads);
-    }
-
-    if (isWrite) {
-      System.out.println("Starting to write data...");
-      writerThreads.start(startKey, endKey, numWriterThreads);
-    }
-
-    if (isUpdate) {
-      LOG.info("Starting to mutate data...");
-      System.out.println("Starting to mutate data...");
-      // TODO : currently append and increment operations not tested with tags
-      // Will update this aftet it is done
-      updaterThreads.start(startKey, endKey, numUpdaterThreads);
-    }
-
-    if (isRead) {
-      System.out.println("Starting to read data...");
-      readerThreads.start(startKey, endKey, numReaderThreads);
-    }
-
-    if (isWrite) {
-      writerThreads.waitForFinish();
-    }
-
-    if (isUpdate) {
-      updaterThreads.waitForFinish();
-    }
-
-    if (isRead) {
-      readerThreads.waitForFinish();
-    }
-
-    boolean success = true;
-    if (isWrite) {
-      success = success && writerThreads.getNumWriteFailures() == 0;
-    }
-    if (isUpdate) {
-      success = success && updaterThreads.getNumWriteFailures() == 0;
-    }
-    if (isRead) {
-      success = success && readerThreads.getNumReadErrors() == 0
-          && readerThreads.getNumReadFailures() == 0;
-    }
-    return success ? EXIT_SUCCESS : EXIT_FAILURE;
-  }
-
-  private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException {
-    try {
-      Class<?> clazz = Class.forName(clazzName);
-      Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class,
-          byte[][].class);
-      return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize,
-          minColsPerKey, maxColsPerKey, families);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-  }
-
-  private MultiThreadedWriter getMultiThreadedWriterInstance(String clazzName
-      , LoadTestDataGenerator dataGen) throws IOException {
-    try {
-      Class<?> clazz = Class.forName(clazzName);
-      Constructor<?> constructor = clazz.getConstructor(
-        LoadTestDataGenerator.class, Configuration.class, TableName.class);
-      return (MultiThreadedWriter) constructor.newInstance(dataGen, conf, tableName);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-  }
-
-  private MultiThreadedUpdater getMultiThreadedUpdaterInstance(String clazzName
-      , LoadTestDataGenerator dataGen) throws IOException {
-    try {
-      Class<?> clazz = Class.forName(clazzName);
-      Constructor<?> constructor = clazz.getConstructor(
-        LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class);
-      return (MultiThreadedUpdater) constructor.newInstance(
-        dataGen, conf, tableName, updatePercent);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-  }
-
-  private MultiThreadedReader getMultiThreadedReaderInstance(String clazzName
-      , LoadTestDataGenerator dataGen) throws IOException {
-    try {
-      Class<?> clazz = Class.forName(clazzName);
-      Constructor<?> constructor = clazz.getConstructor(
-        LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class);
-      return (MultiThreadedReader) constructor.newInstance(dataGen, conf, tableName, verifyPercent);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-  }
-
-  public static byte[] generateData(final Random r, int length) {
-    byte [] b = new byte [length];
-    int i = 0;
-
-    for(i = 0; i < (length-8); i += 8) {
-      b[i] = (byte) (65 + r.nextInt(26));
-      b[i+1] = b[i];
-      b[i+2] = b[i];
-      b[i+3] = b[i];
-      b[i+4] = b[i];
-      b[i+5] = b[i];
-      b[i+6] = b[i];
-      b[i+7] = b[i];
-    }
-
-    byte a = (byte) (65 + r.nextInt(26));
-    for(; i < length; i++) {
-      b[i] = a;
-    }
-    return b;
-  }
-  public static void main(String[] args) {
-    new LoadTestTool().doStaticMain(args);
-  }
-
-  /**
-   * When NUM_TABLES is specified, the function starts multiple worker threads
-   * which individually start a LoadTestTool instance to load a table. Each
-   * table name is in format <tn>_<index>. For example, "-tn test -num_tables 2"
-   * , table names will be "test_1", "test_2"
-   *
-   * @throws IOException
-   */
-  private int parallelLoadTables()
-      throws IOException {
-    // create new command args
-    String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
-    String[] newArgs = null;
-    if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
-      newArgs = new String[cmdLineArgs.length + 2];
-      newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
-      newArgs[1] = LoadTestTool.DEFAULT_TABLE_NAME;
-      System.arraycopy(cmdLineArgs, 0, newArgs, 2, cmdLineArgs.length);
-    } else {
-      newArgs = cmdLineArgs;
-    }
-
-    int tableNameValueIndex = -1;
-    for (int j = 0; j < newArgs.length; j++) {
-      if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
-        tableNameValueIndex = j + 1;
-      } else if (newArgs[j].endsWith(NUM_TABLES)) {
-        // change NUM_TABLES to 1 so that each worker loads one table
-        newArgs[j + 1] = "1";
-      }
-    }
-
-    // starting to load multiple tables
-    List<WorkerThread> workers = new ArrayList<>();
-    for (int i = 0; i < numTables; i++) {
-      String[] workerArgs = newArgs.clone();
-      workerArgs[tableNameValueIndex] = tableName + "_" + (i+1);
-      WorkerThread worker = new WorkerThread(i, workerArgs);
-      workers.add(worker);
-      LOG.info(worker + " starting");
-      worker.start();
-    }
-
-    // wait for all workers finish
-    LOG.info("Waiting for worker threads to finish");
-    for (WorkerThread t : workers) {
-      try {
-        t.join();
-      } catch (InterruptedException ie) {
-        IOException iie = new InterruptedIOException();
-        iie.initCause(ie);
-        throw iie;
-      }
-      checkForErrors();
-    }
-
-    return EXIT_SUCCESS;
-  }
-
-  // If an exception is thrown by one of worker threads, it will be
-  // stored here.
-  protected AtomicReference<Throwable> thrown = new AtomicReference<>();
-
-  private void workerThreadError(Throwable t) {
-    thrown.compareAndSet(null, t);
-  }
-
-  /**
-   * Check for errors in the writer threads. If any is found, rethrow it.
-   */
-  private void checkForErrors() throws IOException {
-    Throwable thrown = this.thrown.get();
-    if (thrown == null) return;
-    if (thrown instanceof IOException) {
-      throw (IOException) thrown;
-    } else {
-      throw new RuntimeException(thrown);
-    }
-  }
-
-  class WorkerThread extends Thread {
-    private String[] workerArgs;
-
-    WorkerThread(int i, String[] args) {
-      super("WorkerThread-" + i);
-      workerArgs = args;
-    }
-
-    @Override
-    public void run() {
-      try {
-        int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
-        if (ret != 0) {
-          throw new RuntimeException("LoadTestTool exit with non-zero return code.");
-        }
-      } catch (Exception ex) {
-        LOG.error("Error in worker thread", ex);
-        workerThreadError(ex);
-      }
-    }
-  }
-
-  private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner,
-      String userList) throws IOException {
-    List<String> users = new ArrayList(Arrays.asList(userList.split(",")));
-    users.add(owner);
-    for (String user : users) {
-      String keyTabFileConfKey = "hbase." + user + ".keytab.file";
-      String principalConfKey = "hbase." + user + ".kerberos.principal";
-      if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey)) {
-        throw new IOException("Authentication configs missing for user : " + user);
-      }
-    }
-    for (String key : authConfig.stringPropertyNames()) {
-      conf.set(key, authConfig.getProperty(key));
-    }
-    LOG.debug("Added authentication properties to config successfully.");
-  }
-
-  public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
-      throws IOException {
-    String hostname = InetAddress.getLocalHost().getHostName();
-    String keyTabFileConfKey = "hbase." + username + ".keytab.file";
-    String keyTabFileLocation = conf.get(keyTabFileConfKey);
-    String principalConfKey = "hbase." + username + ".kerberos.principal";
-    String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname);
-    if (keyTabFileLocation == null || principal == null) {
-      LOG.warn("Principal or key tab file null for : " + principalConfKey + ", "
-          + keyTabFileConfKey);
-    }
-    UserGroupInformation ugi =
-        UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
-    return ugi;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
index 1d2e9a6..6550baa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.util.StringUtils;
 
 /**
  * Common base class for reader and writer parts of multi-thread HBase load
- * test ({@link LoadTestTool}).
+ * test (See LoadTestTool).
  */
 public abstract class MultiThreadedAction {
   private static final Log LOG = LogFactory.getLog(MultiThreadedAction.class);

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
index 1e7e341..e951175 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -121,7 +122,7 @@ public class MultiThreadedReaderWithACL extends MultiThreadedReader {
         UserGroupInformation realUserUgi;
         if(!users.containsKey(userNames[mod])) {
           if(User.isHBaseSecurityEnabled(conf)) {
-            realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]);
+            realUserUgi = HBaseKerberosUtils.loginAndReturnUGI(conf, userNames[mod]);
           } else {
             realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
           }

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
index 40e23fb..9d9bb63 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -138,7 +139,7 @@ public class MultiThreadedUpdaterWithACL extends MultiThreadedUpdater {
         try {
           if (!users.containsKey(userNames[mod])) {
             if (User.isHBaseSecurityEnabled(conf)) {
-              realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]);
+              realUserUgi = HBaseKerberosUtils.loginAndReturnUGI(conf, userNames[mod]);
             } else {
               realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]);
             }

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
index 6beb2e6..7972855 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java
@@ -81,7 +81,7 @@ public class RestartMetaTest extends AbstractHBaseTool {
     // start the writers
     LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(
       minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey,
-      LoadTestTool.DEFAULT_COLUMN_FAMILY);
+      HFileTestUtil.DEFAULT_COLUMN_FAMILY);
     MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME);
     writer.setMultiPut(true);
     writer.start(startKey, endKey, numThreads);
@@ -101,7 +101,7 @@ public class RestartMetaTest extends AbstractHBaseTool {
 
     // create tables if needed
     HBaseTestingUtility.createPreSplitLoadTestTable(conf, TABLE_NAME,
-        LoadTestTool.DEFAULT_COLUMN_FAMILY, Compression.Algorithm.NONE,
+        HFileTestUtil.DEFAULT_COLUMN_FAMILY, Compression.Algorithm.NONE,
         DataBlockEncoding.NONE);
 
     LOG.debug("Loading data....\n\n");
@@ -143,8 +143,8 @@ public class RestartMetaTest extends AbstractHBaseTool {
   @Override
   protected void addOptions() {
     addOptWithArg(OPT_NUM_RS, "Number of Region Servers");
-    addOptWithArg(LoadTestTool.OPT_DATA_BLOCK_ENCODING,
-        LoadTestTool.OPT_DATA_BLOCK_ENCODING_USAGE);
+    addOptWithArg(HFileTestUtil.OPT_DATA_BLOCK_ENCODING,
+        HFileTestUtil.OPT_DATA_BLOCK_ENCODING_USAGE);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java
deleted file mode 100644
index 51dc238..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.apache.hadoop.hbase.HConstants.DEFAULT_REGIONSERVER_PORT;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
-
-@Category({MiscTests.class, SmallTests.class})
-public class TestRegionSizeCalculator {
-
-  private Configuration configuration = new Configuration();
-  private final long megabyte = 1024L * 1024L;
-  private final ServerName sn = ServerName.valueOf("local-rs", DEFAULT_REGIONSERVER_PORT,
-      ServerName.NON_STARTCODE);
-
-  @Test
-  public void testSimpleTestCase() throws Exception {
-
-    RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3");
-
-    Admin admin = mockAdmin(
-        mockRegion("region1", 123),
-        mockRegion("region3", 1232),
-        mockRegion("region2",  54321)
-    );
-
-    RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin);
-
-    assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes()));
-    assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes()));
-    assertEquals(1232 * megabyte, calculator.getRegionSize("region3".getBytes()));
-    // if regionCalculator does not know about a region, it should return 0
-    assertEquals(0 * megabyte, calculator.getRegionSize("otherTableRegion".getBytes()));
-
-    assertEquals(3, calculator.getRegionSizeMap().size());
-  }
-
-
-  /**
-   * When size of region in megabytes is larger than largest possible integer there could be
-   * error caused by lost of precision.
-   * */
-  @Test
-  public void testLargeRegion() throws Exception {
-
-    RegionLocator regionLocator = mockRegionLocator("largeRegion");
-
-    Admin admin = mockAdmin(
-        mockRegion("largeRegion", Integer.MAX_VALUE)
-    );
-
-    RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin);
-
-    assertEquals(((long) Integer.MAX_VALUE) * megabyte, calculator.getRegionSize("largeRegion".getBytes()));
-  }
-
-  /** When calculator is disabled, it should return 0 for each request.*/
-  @Test
-  public void testDisabled() throws Exception {
-    String regionName = "cz.goout:/index.html";
-    RegionLocator table = mockRegionLocator(regionName);
-
-    Admin admin = mockAdmin(
-        mockRegion(regionName, 999)
-    );
-
-    //first request on enabled calculator
-    RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin);
-    assertEquals(999 * megabyte, calculator.getRegionSize(regionName.getBytes()));
-
-    //then disabled calculator.
-    configuration.setBoolean(RegionSizeCalculator.ENABLE_REGIONSIZECALCULATOR, false);
-    RegionSizeCalculator disabledCalculator = new RegionSizeCalculator(table, admin);
-    assertEquals(0 * megabyte, disabledCalculator.getRegionSize(regionName.getBytes()));
-
-    assertEquals(0, disabledCalculator.getRegionSizeMap().size());
-  }
-
-  /**
-   * Makes some table with given region names.
-   * */
-  private RegionLocator mockRegionLocator(String... regionNames) throws IOException {
-    RegionLocator mockedTable = Mockito.mock(RegionLocator.class);
-    when(mockedTable.getName()).thenReturn(TableName.valueOf("sizeTestTable"));
-    List<HRegionLocation> regionLocations = new ArrayList<>(regionNames.length);
-    when(mockedTable.getAllRegionLocations()).thenReturn(regionLocations);
-
-    for (String regionName : regionNames) {
-      HRegionInfo info = Mockito.mock(HRegionInfo.class);
-      when(info.getRegionName()).thenReturn(regionName.getBytes());
-      regionLocations.add(new HRegionLocation(info, sn));
-    }
-
-    return mockedTable;
-  }
-
-  /**
-   * Creates mock returning RegionLoad info about given servers.
-  */
-  private Admin mockAdmin(RegionLoad... regionLoadArray) throws Exception {
-    Admin mockAdmin = Mockito.mock(Admin.class);
-    Map<byte[], RegionLoad> regionLoads = new TreeMap<>(Bytes.BYTES_COMPARATOR);
-    for (RegionLoad regionLoad : regionLoadArray) {
-      regionLoads.put(regionLoad.getName(), regionLoad);
-    }
-    when(mockAdmin.getConfiguration()).thenReturn(configuration);
-    when(mockAdmin.getRegionLoad(sn, TableName.valueOf("sizeTestTable"))).thenReturn(regionLoads);
-    return mockAdmin;
-  }
-
-  /**
-   * Creates mock of region with given name and size.
-   *
-   * @param  fileSizeMb number of megabytes occupied by region in file store in megabytes
-   * */
-  private RegionLoad mockRegion(String regionName, int fileSizeMb) {
-    RegionLoad region = Mockito.mock(RegionLoad.class);
-    when(region.getName()).thenReturn(regionName.getBytes());
-    when(region.getNameAsString()).thenReturn(regionName);
-    when(region.getStorefileSizeMB()).thenReturn(fileSizeMb);
-    return region;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
index bf7bf45..2deba00 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
@@ -17,6 +17,7 @@
 package org.apache.hadoop.hbase.util.test;
 
 import java.io.IOException;
+import java.util.Random;
 import java.util.Set;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -56,6 +57,28 @@ public abstract class LoadTestDataGenerator {
     this.kvGenerator = new LoadTestKVGenerator(minValueSize, maxValueSize);
   }
 
+  public static byte[] generateData(final Random r, int length) {
+    byte [] b = new byte [length];
+    int i = 0;
+
+    for(i = 0; i < (length-8); i += 8) {
+      b[i] = (byte) (65 + r.nextInt(26));
+      b[i+1] = b[i];
+      b[i+2] = b[i];
+      b[i+3] = b[i];
+      b[i+4] = b[i];
+      b[i+5] = b[i];
+      b[i+6] = b[i];
+      b[i+7] = b[i];
+    }
+
+    byte a = (byte) (65 + r.nextInt(26));
+    for(; i < length; i++) {
+      b[i] = a;
+    }
+    return b;
+  }
+
   /**
    * initialize the LoadTestDataGenerator
    *

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties b/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
deleted file mode 100644
index 6fca96a..0000000
--- a/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-# ResourceBundle properties file for Map-Reduce counters
-
-#/**
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements.  See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership.  The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License.  You may obtain a copy of the License at
-# *
-# *     http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-CounterGroupName=              HBase Performance Evaluation
-ELAPSED_TIME.name=             Elapsed time in milliseconds
-ROWS.name=									  Row count
-# ResourceBundle properties file for Map-Reduce counters
-
-CounterGroupName=              HBase Performance Evaluation
-ELAPSED_TIME.name=             Elapsed time in milliseconds
-ROWS.name=									  Row count

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format b/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format
deleted file mode 100755
index 762ddd7..0000000
Binary files a/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-spark/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml
index aa47947..3761d59 100644
--- a/hbase-spark/pom.xml
+++ b/hbase-spark/pom.xml
@@ -492,6 +492,10 @@
       <type>test-jar</type>
     </dependency>
     <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-mapreduce</artifactId>
+    </dependency>
+    <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java
----------------------------------------------------------------------
diff --git a/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java b/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java
index 93cd939..bfacbe8 100644
--- a/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java
+++ b/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java
@@ -27,7 +27,6 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index be0be99..f973cd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,6 +62,7 @@
 
   <modules>
     <module>hbase-replication</module>
+    <module>hbase-mapreduce</module>
     <module>hbase-resource-bundle</module>
     <module>hbase-server</module>
     <module>hbase-thrift</module>
@@ -1582,6 +1583,18 @@
         <scope>test</scope>
       </dependency>
       <dependency>
+        <artifactId>hbase-mapreduce</artifactId>
+        <groupId>org.apache.hbase</groupId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <artifactId>hbase-mapreduce</artifactId>
+        <groupId>org.apache.hbase</groupId>
+        <version>${project.version}</version>
+        <type>test-jar</type>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
         <artifactId>hbase-endpoint</artifactId>
         <groupId>org.apache.hbase</groupId>
         <version>${project.version}</version>

http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index 6181b13..f96cd6c 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -2478,7 +2478,7 @@ void rename(Admin admin, String oldTableName, TableName newTableName) {
 RegionServer Grouping (A.K.A `rsgroup`) is an advanced feature for
 partitioning regionservers into distinctive groups for strict isolation. It
 should only be used by users who are sophisticated enough to understand the
-full implications and have a sufficient background in managing HBase clusters. 
+full implications and have a sufficient background in managing HBase clusters.
 It was developed by Yahoo! and they run it at scale on their large grid cluster.
 See link:http://www.slideshare.net/HBaseCon/keynote-apache-hbase-at-yahoo-scale[HBase at Yahoo! Scale].
 
@@ -2491,20 +2491,20 @@ rsgroup at a time. By default, all tables and regionservers belong to the
 APIs. A custom balancer implementation tracks assignments per rsgroup and makes
 sure to move regions to the relevant regionservers in that rsgroup. The rsgroup
 information is stored in a regular HBase table, and a zookeeper-based read-only
-cache is used at cluster bootstrap time. 
+cache is used at cluster bootstrap time.
 
-To enable, add the following to your hbase-site.xml and restart your Master: 
+To enable, add the following to your hbase-site.xml and restart your Master:
 
 [source,xml]
 ----
- <property> 
-   <name>hbase.coprocessor.master.classes</name> 
-   <value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value> 
- </property> 
- <property> 
-   <name>hbase.master.loadbalancer.class</name> 
-   <value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value> 
- </property> 
+ <property>
+   <name>hbase.coprocessor.master.classes</name>
+   <value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value>
+ </property>
+ <property>
+   <name>hbase.master.loadbalancer.class</name>
+   <value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value>
+ </property>
 ----
 
 Then use the shell _rsgroup_ commands to create and manipulate RegionServer
@@ -2514,7 +2514,7 @@ rsgroup commands available in the hbase shell type:
 [source, bash]
 ----
  hbase(main):008:0> help ‘rsgroup’
- Took 0.5610 seconds 
+ Took 0.5610 seconds
 ----
 
 High level, you create a rsgroup that is other than the `default` group using
@@ -2531,8 +2531,8 @@ Here is example using a few of the rsgroup  commands. To add a group, do as foll
 
 [source, bash]
 ----
- hbase(main):008:0> add_rsgroup 'my_group' 
- Took 0.5610 seconds 
+ hbase(main):008:0> add_rsgroup 'my_group'
+ Took 0.5610 seconds
 ----
 
 
@@ -2556,11 +2556,11 @@ ERROR: org.apache.hadoop.hbase.exceptions.UnknownProtocolException: No registere
 ====
 
 Add a server (specified by hostname + port) to the just-made group using the
-_move_servers_rsgroup_ command as follows: 
+_move_servers_rsgroup_ command as follows:
 
 [source, bash]
 ----
- hbase(main):010:0> move_servers_rsgroup 'my_group',['k.att.net:51129'] 
+ hbase(main):010:0> move_servers_rsgroup 'my_group',['k.att.net:51129']
 ----
 
 .Hostname and Port vs ServerName


Mime
View raw message