hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [41/50] [abbrv] hbase git commit: HBASE-14678 Experiment: Temporarily disable balancer and a few others to see if root of crashed/timedout JVMs
Date Fri, 23 Oct 2015 22:50:01 GMT
HBASE-14678 Experiment: Temporarily disable balancer and a few others to see if root of crashed/timedout JVMs


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93023f54
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93023f54
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93023f54

Branch: refs/heads/hbase-12439
Commit: 93023f544b673ccc99fc0e327f2eca8964128097
Parents: 35660b4
Author: stack <stack@apache.org>
Authored: Thu Oct 22 12:31:04 2015 -0700
Committer: stack <stack@apache.org>
Committed: Thu Oct 22 12:31:04 2015 -0700

----------------------------------------------------------------------
 .../hbase/TestPartialResultsFromClientSide.java | 832 -------------------
 .../balancer/TestStochasticLoadBalancer.java    | 532 ------------
 .../balancer/TestStochasticLoadBalancer2.java   |  90 --
 .../TestMasterFailoverWithProcedures.java       | 514 ------------
 .../hbase/client/TestReplicationShell.java      |  37 -
 .../apache/hadoop/hbase/client/TestShell.java   |  39 -
 6 files changed, 2044 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
deleted file mode 100644
index 3794e59..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPartialResultsFromClientSide.java
+++ /dev/null
@@ -1,832 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.client.ClientScanner;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
-import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
-import org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter;
-import org.apache.hadoop.hbase.filter.RandomRowFilter;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * These tests are focused on testing how partial results appear to a client. Partial results are
- * {@link Result}s that contain only a portion of a row's complete list of cells. Partial results
- * are formed when the server breaches its maximum result size when trying to service a client's RPC
- * request. It is the responsibility of the scanner on the client side to recognize when partial
- * results have been returned and to take action to form the complete results.
- * <p>
- * Unless the flag {@link Scan#setAllowPartialResults(boolean)} has been set to true, the caller of
- * {@link ResultScanner#next()} should never see partial results.
- */
-@Category(MediumTests.class)
-public class TestPartialResultsFromClientSide {
-  private static final Log LOG = LogFactory.getLog(TestPartialResultsFromClientSide.class);
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
-  private static Table TABLE = null;
-
-  /**
-   * Table configuration
-   */
-  private static TableName TABLE_NAME = TableName.valueOf("testTable");
-
-  private static int NUM_ROWS = 5;
-  private static byte[] ROW = Bytes.toBytes("testRow");
-  private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
-
-  // Should keep this value below 10 to keep generation of expected kv's simple. If above 10 then
-  // table/row/cf1/... will be followed by table/row/cf10/... instead of table/row/cf2/... which
-  // breaks the simple generation of expected kv's
-  private static int NUM_FAMILIES = 10;
-  private static byte[] FAMILY = Bytes.toBytes("testFamily");
-  private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
-
-  private static int NUM_QUALIFIERS = 10;
-  private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
-  private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
-
-  private static int VALUE_SIZE = 1024;
-  private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
-
-  private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
-
-  // Approximation of how large the heap size of cells in our table. Should be accessed through
-  // getCellHeapSize().
-  private static long CELL_HEAP_SIZE = -1;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.startMiniCluster(3);
-    TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
-  }
-
-  static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
-      byte[][] qualifiers, byte[] cellValue) throws IOException {
-    Table ht = TEST_UTIL.createTable(name, families);
-    List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
-    ht.put(puts);
-
-    return ht;
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  /**
-   * Ensure that the expected key values appear in a result returned from a scanner that is
-   * combining partial results into complete results
-   * @throws Exception
-   */
-  @Test
-  public void testExpectedValuesOfPartialResults() throws Exception {
-    testExpectedValuesOfPartialResults(false);
-    testExpectedValuesOfPartialResults(true);
-  }
-
-  public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception {
-    Scan partialScan = new Scan();
-    partialScan.setMaxVersions();
-    // Max result size of 1 ensures that each RPC request will return a single cell. The scanner
-    // will need to reconstruct the results into a complete result before returning to the caller
-    partialScan.setMaxResultSize(1);
-    partialScan.setReversed(reversed);
-    ResultScanner partialScanner = TABLE.getScanner(partialScan);
-
-    final int startRow = reversed ? ROWS.length - 1 : 0;
-    final int endRow = reversed ? -1 : ROWS.length;
-    final int loopDelta = reversed ? -1 : 1;
-    String message;
-
-    for (int row = startRow; row != endRow; row = row + loopDelta) {
-      message = "Ensuring the expected keyValues are present for row " + row;
-      List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE);
-      Result result = partialScanner.next();
-      assertFalse(result.isPartial());
-      verifyResult(result, expectedKeyValues, message);
-    }
-
-    partialScanner.close();
-  }
-
-  /**
-   * Ensure that we only see Results marked as partial when the allowPartial flag is set
-   * @throws Exception
-   */
-  @Test
-  public void testAllowPartialResults() throws Exception {
-    Scan scan = new Scan();
-    scan.setAllowPartialResults(true);
-    scan.setMaxResultSize(1);
-    ResultScanner scanner = TABLE.getScanner(scan);
-    Result result = scanner.next();
-
-    assertTrue(result != null);
-    assertTrue(result.isPartial());
-    assertTrue(result.rawCells() != null);
-    assertTrue(result.rawCells().length == 1);
-
-    scanner.close();
-
-    scan.setAllowPartialResults(false);
-    scanner = TABLE.getScanner(scan);
-    result = scanner.next();
-
-    assertTrue(result != null);
-    assertTrue(!result.isPartial());
-    assertTrue(result.rawCells() != null);
-    assertTrue(result.rawCells().length == NUM_COLS);
-
-    scanner.close();
-  }
-
-  /**
-   * Ensure that the results returned from a scanner that retrieves all results in a single RPC call
-   * matches the results that are returned from a scanner that must incrementally combine partial
-   * results into complete results. A variety of scan configurations can be tested
-   * @throws Exception
-   */
-  @Test
-  public void testEquivalenceOfScanResults() throws Exception {
-    Scan oneShotScan = new Scan();
-    oneShotScan.setMaxResultSize(Long.MAX_VALUE);
-
-    Scan partialScan = new Scan(oneShotScan);
-    partialScan.setMaxResultSize(1);
-
-    testEquivalenceOfScanResults(TABLE, oneShotScan, partialScan);
-  }
-
-  public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception {
-    ResultScanner scanner1 = table.getScanner(scan1);
-    ResultScanner scanner2 = table.getScanner(scan2);
-
-    Result r1 = null;
-    Result r2 = null;
-    int count = 0;
-
-    while ((r1 = scanner1.next()) != null) {
-      r2 = scanner2.next();
-
-      assertTrue(r2 != null);
-      compareResults(r1, r2, "Comparing result #" + count);
-      count++;
-    }
-
-    r2 = scanner2.next();
-    assertTrue("r2: " + r2 + " Should be null", r2 == null);
-
-    scanner1.close();
-    scanner2.close();
-  }
-
-  /**
-   * Order of cells in partial results matches the ordering of cells from complete results
-   * @throws Exception
-   */
-  @Test
-  public void testOrderingOfCellsInPartialResults() throws Exception {
-    Scan scan = new Scan();
-
-    for (int col = 1; col <= NUM_COLS; col++) {
-      scan.setMaxResultSize(getResultSizeForNumberOfCells(col));
-      testOrderingOfCellsInPartialResults(scan);
-
-      // Test again with a reversed scanner
-      scan.setReversed(true);
-      testOrderingOfCellsInPartialResults(scan);
-    }
-  }
-
-  public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception {
-    // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true
-    // the results will NOT be reconstructed and instead the caller will see the partial results
-    // returned by the server
-    Scan partialScan = new Scan(basePartialScan);
-    partialScan.setAllowPartialResults(true);
-    ResultScanner partialScanner = TABLE.getScanner(partialScan);
-
-    // Scan that retrieves all table results in single RPC request
-    Scan oneShotScan = new Scan(basePartialScan);
-    oneShotScan.setMaxResultSize(Long.MAX_VALUE);
-    oneShotScan.setCaching(ROWS.length);
-    ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
-
-    Result oneShotResult = oneShotScanner.next();
-    Result partialResult = null;
-    int iterationCount = 0;
-
-    while (oneShotResult != null && oneShotResult.rawCells() != null) {
-      List<Cell> aggregatePartialCells = new ArrayList<Cell>();
-      do {
-        partialResult = partialScanner.next();
-        assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null);
-        assertTrue("Partial cells are null. iteration: " + iterationCount,
-          partialResult.rawCells() != null);
-
-        for (Cell c : partialResult.rawCells()) {
-          aggregatePartialCells.add(c);
-        }
-      } while (partialResult.isPartial());
-
-      assertTrue("Number of cells differs. iteration: " + iterationCount,
-        oneShotResult.rawCells().length == aggregatePartialCells.size());
-      final Cell[] oneShotCells = oneShotResult.rawCells();
-      for (int cell = 0; cell < oneShotCells.length; cell++) {
-        Cell oneShotCell = oneShotCells[cell];
-        Cell partialCell = aggregatePartialCells.get(cell);
-
-        assertTrue("One shot cell was null", oneShotCell != null);
-        assertTrue("Partial cell was null", partialCell != null);
-        assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell,
-          oneShotCell.equals(partialCell));
-      }
-
-      oneShotResult = oneShotScanner.next();
-      iterationCount++;
-    }
-
-    assertTrue(partialScanner.next() == null);
-
-    partialScanner.close();
-    oneShotScanner.close();
-  }
-
-  /**
-   * Setting the max result size allows us to control how many cells we expect to see on each call
-   * to next on the scanner. Test a variety of different sizes for correctness
-   * @throws Exception
-   */
-  @Test
-  public void testExpectedNumberOfCellsPerPartialResult() throws Exception {
-    Scan scan = new Scan();
-    testExpectedNumberOfCellsPerPartialResult(scan);
-
-    scan.setReversed(true);
-    testExpectedNumberOfCellsPerPartialResult(scan);
-  }
-
-  public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan) throws Exception {
-    for (int expectedCells = 1; expectedCells <= NUM_COLS; expectedCells++) {
-      testExpectedNumberOfCellsPerPartialResult(baseScan, expectedCells);
-    }
-  }
-
-  public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells)
-      throws Exception {
-
-    if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells);
-
-    // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back
-    // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less
-    // than it in cases where expectedNumberOfCells is not an exact multiple of the number of
-    // columns in the table.
-    Scan scan = new Scan(baseScan);
-    scan.setAllowPartialResults(true);
-    scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells));
-
-    ResultScanner scanner = TABLE.getScanner(scan);
-    Result result = null;
-    byte[] prevRow = null;
-    while ((result = scanner.next()) != null) {
-      assertTrue(result.rawCells() != null);
-
-      // Cases when cell count won't equal expectedNumberOfCells:
-      // 1. Returned result is the final result needed to form the complete result for that row
-      // 2. It is the first result we have seen for that row and thus may have been fetched as
-      // the last group of cells that fit inside the maxResultSize
-      assertTrue(
-        "Result's cell count differed from expected number. result: " + result,
-        result.rawCells().length == expectedNumberOfCells || !result.isPartial()
-          || !Bytes.equals(prevRow, result.getRow()));
-      prevRow = result.getRow();
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * @return The approximate heap size of a cell in the test table. All cells should have
-   *         approximately the same heap size, so the value is cached to avoid repeating the
-   *         calculation
-   * @throws Exception
-   */
-  private long getCellHeapSize() throws Exception {
-    if (CELL_HEAP_SIZE == -1) {
-      // Do a partial scan that will return a single result with a single cell
-      Scan scan = new Scan();
-      scan.setMaxResultSize(1);
-      scan.setAllowPartialResults(true);
-      ResultScanner scanner = TABLE.getScanner(scan);
-
-      Result result = scanner.next();
-
-      assertTrue(result != null);
-      assertTrue(result.rawCells() != null);
-      assertTrue(result.rawCells().length == 1);
-
-      CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]);
-      if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE);
-      scanner.close();
-    }
-
-    return CELL_HEAP_SIZE;
-  }
-
-  /**
-   * @param numberOfCells
-   * @return the result size that should be used in {@link Scan#setMaxResultSize(long)} if you want
-   *         the server to return exactly numberOfCells cells
-   * @throws Exception
-   */
-  private long getResultSizeForNumberOfCells(int numberOfCells) throws Exception {
-    return getCellHeapSize() * numberOfCells;
-  }
-
-  /**
-   * Test various combinations of batching and partial results for correctness
-   */
-  @Test
-  public void testPartialResultsAndBatch() throws Exception {
-    for (int batch = 1; batch <= NUM_COLS / 4; batch++) {
-      for (int cellsPerPartial = 1; cellsPerPartial <= NUM_COLS / 4; cellsPerPartial++) {
-        testPartialResultsAndBatch(batch, cellsPerPartial);
-      }
-    }
-  }
-
-  public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)
-      throws Exception {
-    if (LOG.isInfoEnabled()) {
-      LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult);
-    }
-
-    Scan scan = new Scan();
-    scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult));
-    scan.setBatch(batch);
-    ResultScanner scanner = TABLE.getScanner(scan);
-    Result result = scanner.next();
-    int repCount = 0;
-
-    while ((result = scanner.next()) != null) {
-      assertTrue(result.rawCells() != null);
-
-      if (result.isPartial()) {
-        final String error =
-            "Cells:" + result.rawCells().length + " Batch size:" + batch
-                + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount;
-        assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult));
-      } else {
-        assertTrue(result.rawCells().length <= batch);
-      }
-      repCount++;
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * Test the method {@link Result#createCompleteResult(List, Result)}
-   * @throws Exception
-   */
-  @Test
-  public void testPartialResultsReassembly() throws Exception {
-    Scan scan = new Scan();
-    testPartialResultsReassembly(scan);
-    scan.setReversed(true);
-    testPartialResultsReassembly(scan);
-  }
-
-  public void testPartialResultsReassembly(Scan scanBase) throws Exception {
-    Scan partialScan = new Scan(scanBase);
-    partialScan.setMaxResultSize(1);
-    partialScan.setAllowPartialResults(true);
-    ResultScanner partialScanner = TABLE.getScanner(partialScan);
-
-    Scan oneShotScan = new Scan(scanBase);
-    oneShotScan.setMaxResultSize(Long.MAX_VALUE);
-    ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);
-
-    ArrayList<Result> partials = new ArrayList<>();
-    for (int i = 0; i < NUM_ROWS; i++) {
-      Result partialResult = null;
-      Result completeResult = null;
-      Result oneShotResult = null;
-      partials.clear();
-
-      do {
-        partialResult = partialScanner.next();
-        partials.add(partialResult);
-      } while (partialResult != null && partialResult.isPartial());
-
-      completeResult = Result.createCompleteResult(partials);
-      oneShotResult = oneShotScanner.next();
-
-      compareResults(completeResult, oneShotResult, null);
-    }
-
-    assertTrue(oneShotScanner.next() == null);
-    assertTrue(partialScanner.next() == null);
-
-    oneShotScanner.close();
-    partialScanner.close();
-  }
-
-  /**
-   * When reconstructing the complete result from its partials we ensure that the row of each
-   * partial result is the same. If one of the rows differs, an exception is thrown.
-   */
-  @Test
-  public void testExceptionThrownOnMismatchedPartialResults() throws IOException {
-    assertTrue(NUM_ROWS >= 2);
-
-    ArrayList<Result> partials = new ArrayList<>();
-    Scan scan = new Scan();
-    scan.setMaxResultSize(Long.MAX_VALUE);
-    ResultScanner scanner = TABLE.getScanner(scan);
-    Result r1 = scanner.next();
-    partials.add(r1);
-    Result r2 = scanner.next();
-    partials.add(r2);
-
-    assertFalse(Bytes.equals(r1.getRow(), r2.getRow()));
-
-    try {
-      Result.createCompleteResult(partials);
-      fail("r1 and r2 are from different rows. It should not be possible to combine them into"
-          + " a single result");
-    } catch (IOException e) {
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is
-   * true, the scanner should not return partial results. The scanner cannot return partial results
-   * because the entire row needs to be read for the include/exclude decision to be made
-   */
-  @Test
-  public void testNoPartialResultsWhenRowFilterPresent() throws Exception {
-    Scan scan = new Scan();
-    scan.setMaxResultSize(1);
-    scan.setAllowPartialResults(true);
-    // If a filter hasFilter() is true then partial results should not be returned else filter
-    // application server side would break.
-    scan.setFilter(new RandomRowFilter(1.0f));
-    ResultScanner scanner = TABLE.getScanner(scan);
-
-    Result r = null;
-    while ((r = scanner.next()) != null) {
-      assertFalse(r.isPartial());
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * Examine the interaction between the maxResultSize and caching. If the caching limit is reached
-   * before the maxResultSize limit, we should not see partial results. On the other hand, if the
-   * maxResultSize limit is reached before the caching limit, it is likely that partial results will
-   * be seen.
-   * @throws Exception
-   */
-  @Test
-  public void testPartialResultsAndCaching() throws Exception {
-    for (int caching = 1; caching <= NUM_ROWS; caching++) {
-      for (int maxResultRows = 0; maxResultRows <= NUM_ROWS; maxResultRows++) {
-        testPartialResultsAndCaching(maxResultRows, caching);
-      }
-    }
-  }
-
-  /**
-   * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
-   * @param cachingRowLimit The row limit that will be enforced through caching
-   * @throws Exception
-   */
-  public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
-      throws Exception {
-    Scan scan = new Scan();
-    scan.setAllowPartialResults(true);
-
-    // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
-    // ensure that the result size we specify is not an exact multiple of the number of cells
-    // in a row. This ensures that partial results will be returned when the result size limit
-    // is reached before the caching limit.
-    int cellOffset = NUM_COLS / 3;
-    long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
-    scan.setMaxResultSize(maxResultSize);
-    scan.setCaching(cachingRowLimit);
-
-    ResultScanner scanner = TABLE.getScanner(scan);
-    ClientScanner clientScanner = (ClientScanner) scanner;
-    Result r = null;
-
-    // Approximate the number of rows we expect will fit into the specified max rsult size. If this
-    // approximation is less than caching, then we expect that the max result size limit will be
-    // hit before the caching limit and thus partial results may be seen
-    boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
-    while ((r = clientScanner.next()) != null) {
-      assertTrue(!r.isPartial() || expectToSeePartialResults);
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * Small scans should not return partial results because it would prevent small scans from
-   * retrieving all of the necessary results in a single RPC request which is what makese small
-   * scans useful. Thus, ensure that even when {@link Scan#getAllowPartialResults()} is true, small
-   * scans do not return partial results
-   * @throws Exception
-   */
-  @Test
-  public void testSmallScansDoNotAllowPartials() throws Exception {
-    Scan scan = new Scan();
-    testSmallScansDoNotAllowPartials(scan);
-    scan.setReversed(true);
-    testSmallScansDoNotAllowPartials(scan);
-  }
-
-  public void testSmallScansDoNotAllowPartials(Scan baseScan) throws Exception {
-    Scan scan = new Scan(baseScan);
-    scan.setAllowPartialResults(true);
-    scan.setSmall(true);
-    scan.setMaxResultSize(1);
-
-    ResultScanner scanner = TABLE.getScanner(scan);
-    Result r = null;
-
-    while ((r = scanner.next()) != null) {
-      assertFalse(r.isPartial());
-    }
-
-    scanner.close();
-  }
-
-  /**
-   * Make puts to put the input value into each combination of row, family, and qualifier
-   * @param rows
-   * @param families
-   * @param qualifiers
-   * @param value
-   * @return
-   * @throws IOException
-   */
-  static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
-      byte[] value) throws IOException {
-    Put put;
-    ArrayList<Put> puts = new ArrayList<>();
-
-    for (int row = 0; row < rows.length; row++) {
-      put = new Put(rows[row]);
-      for (int fam = 0; fam < families.length; fam++) {
-        for (int qual = 0; qual < qualifiers.length; qual++) {
-          KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
-          put.add(kv);
-        }
-      }
-      puts.add(put);
-    }
-
-    return puts;
-  }
-
-  /**
-   * Make key values to represent each possible combination of family and qualifier in the specified
-   * row.
-   * @param row
-   * @param families
-   * @param qualifiers
-   * @param value
-   * @return
-   */
-  static ArrayList<Cell> createKeyValuesForRow(byte[] row, byte[][] families, byte[][] qualifiers,
-      byte[] value) {
-    ArrayList<Cell> outList = new ArrayList<>();
-    for (int fam = 0; fam < families.length; fam++) {
-      for (int qual = 0; qual < qualifiers.length; qual++) {
-        outList.add(new KeyValue(row, families[fam], qualifiers[qual], qual, value));
-      }
-    }
-    return outList;
-  }
-
-  /**
-   * Verifies that result contains all the key values within expKvList. Fails the test otherwise
-   * @param result
-   * @param expKvList
-   * @param msg
-   */
-  static void verifyResult(Result result, List<Cell> expKvList, String msg) {
-    if (LOG.isInfoEnabled()) {
-      LOG.info(msg);
-      LOG.info("Expected count: " + expKvList.size());
-      LOG.info("Actual count: " + result.size());
-    }
-
-    if (expKvList.size() == 0) return;
-
-    int i = 0;
-    for (Cell kv : result.rawCells()) {
-      if (i >= expKvList.size()) {
-        break; // we will check the size later
-      }
-
-      Cell kvExp = expKvList.get(i++);
-      assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(),
-        kvExp.equals(kv));
-    }
-
-    assertEquals(expKvList.size(), result.size());
-  }
-
-  /**
-   * Compares two results and fails the test if the results are different
-   * @param r1
-   * @param r2
-   * @param message
-   */
-  static void compareResults(Result r1, Result r2, final String message) {
-    if (LOG.isInfoEnabled()) {
-      if (message != null) LOG.info(message);
-      LOG.info("r1: " + r1);
-      LOG.info("r2: " + r2);
-    }
-
-    final String failureMessage = "Results r1:" + r1 + " \nr2:" + r2 + " are not equivalent";
-    if (r1 == null && r2 == null) fail(failureMessage);
-    else if (r1 == null || r2 == null) fail(failureMessage);
-
-    try {
-      Result.compareResults(r1, r2);
-    } catch (Exception e) {
-      fail(failureMessage);
-    }
-  }
-
-  @Test
-  public void testReadPointAndPartialResults() throws Exception {
-    TableName testName = TableName.valueOf("testReadPointAndPartialResults");
-    int numRows = 5;
-    int numFamilies = 5;
-    int numQualifiers = 5;
-    byte[][] rows = HTestConst.makeNAscii(Bytes.toBytes("testRow"), numRows);
-    byte[][] families = HTestConst.makeNAscii(Bytes.toBytes("testFamily"), numFamilies);
-    byte[][] qualifiers = HTestConst.makeNAscii(Bytes.toBytes("testQualifier"), numQualifiers);
-    byte[] value = Bytes.createMaxByteArray(100);
-
-    Table tmpTable = createTestTable(testName, rows, families, qualifiers, value);
-
-    Scan scan = new Scan();
-    scan.setMaxResultSize(1);
-    scan.setAllowPartialResults(true);
-
-    // Open scanner before deletes
-    ResultScanner scanner = tmpTable.getScanner(scan);
-
-    Delete delete1 = new Delete(rows[0]);
-    delete1.addColumn(families[0], qualifiers[0], 0);
-    tmpTable.delete(delete1);
-
-    Delete delete2 = new Delete(rows[1]);
-    delete2.addColumn(families[1], qualifiers[1], 1);
-    tmpTable.delete(delete2);
-
-    // Should see all cells because scanner was opened prior to deletes
-    int scannerCount = countCellsFromScanner(scanner);
-    int expectedCount = numRows * numFamilies * numQualifiers;
-    assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
-      scannerCount == expectedCount);
-
-    // Minus 2 for the two cells that were deleted
-    scanner = tmpTable.getScanner(scan);
-    scannerCount = countCellsFromScanner(scanner);
-    expectedCount = numRows * numFamilies * numQualifiers - 2;
-    assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
-      scannerCount == expectedCount);
-
-    scanner = tmpTable.getScanner(scan);
-    // Put in 2 new rows. The timestamps differ from the deleted rows
-    Put put1 = new Put(rows[0]);
-    put1.add(new KeyValue(rows[0], families[0], qualifiers[0], 1, value));
-    tmpTable.put(put1);
-
-    Put put2 = new Put(rows[1]);
-    put2.add(new KeyValue(rows[1], families[1], qualifiers[1], 2, value));
-    tmpTable.put(put2);
-
-    // Scanner opened prior to puts. Cell count shouldn't have changed
-    scannerCount = countCellsFromScanner(scanner);
-    expectedCount = numRows * numFamilies * numQualifiers - 2;
-    assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
-      scannerCount == expectedCount);
-
-    // Now the scanner should see the cells that were added by puts
-    scanner = tmpTable.getScanner(scan);
-    scannerCount = countCellsFromScanner(scanner);
-    expectedCount = numRows * numFamilies * numQualifiers;
-    assertTrue("scannerCount: " + scannerCount + " expectedCount: " + expectedCount,
-      scannerCount == expectedCount);
-
-    TEST_UTIL.deleteTable(testName);
-  }
-
-  /**
-   * Exhausts the scanner by calling next repetitively. Once completely exhausted, close scanner and
-   * return total cell count
-   * @param scanner
-   * @return
-   * @throws Exception
-   */
-  private int countCellsFromScanner(ResultScanner scanner) throws Exception {
-    Result result = null;
-    int numCells = 0;
-    while ((result = scanner.next()) != null) {
-      numCells += result.rawCells().length;
-    }
-
-    scanner.close();
-    return numCells;
-  }
-
-  /**
-   * Test partial Result re-assembly in the presence of different filters. The Results from the
-   * partial scanner should match the Results returned from a scanner that receives all of the
-   * results in one RPC to the server. The partial scanner is tested with a variety of different
-   * result sizes (all of which are less than the size necessary to fetch an entire row)
-   * @throws Exception
-   */
-  @Test
-  public void testPartialResultsWithColumnFilter() throws Exception {
-    testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter());
-    testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5")));
-    testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true,
-        Bytes.toBytes("testQualifier7"), true));
-
-    Set<byte[]> qualifiers = new LinkedHashSet<>();
-    qualifiers.add(Bytes.toBytes("testQualifier5"));
-    testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers));
-  }
-
-  public void testPartialResultsWithColumnFilter(Filter filter) throws Exception {
-    assertTrue(!filter.hasFilterRow());
-
-    Scan partialScan = new Scan();
-    partialScan.setFilter(filter);
-
-    Scan oneshotScan = new Scan();
-    oneshotScan.setFilter(filter);
-    oneshotScan.setMaxResultSize(Long.MAX_VALUE);
-
-    for (int i = 1; i <= NUM_COLS; i++) {
-      partialScan.setMaxResultSize(getResultSizeForNumberOfCells(i));
-      testEquivalenceOfScanResults(TABLE, partialScan, oneshotScan);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
deleted file mode 100644
index 7abbeb4..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.balancer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Queue;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RegionLoad;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.master.RackManager;
-import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
-import org.apache.hadoop.hbase.testclassification.FlakeyTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({FlakeyTests.class, MediumTests.class})
-public class TestStochasticLoadBalancer extends BalancerTestBase {
-  public static final String REGION_KEY = "testRegion";
-  private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
-
-  @Test
-  public void testKeepRegionLoad() throws Exception {
-
-    ServerName sn = ServerName.valueOf("test:8080", 100);
-    int numClusterStatusToAdd = 20000;
-    for (int i = 0; i < numClusterStatusToAdd; i++) {
-      ServerLoad sl = mock(ServerLoad.class);
-
-      RegionLoad rl = mock(RegionLoad.class);
-      when(rl.getStores()).thenReturn(i);
-
-      Map<byte[], RegionLoad> regionLoadMap =
-          new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
-      regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
-      when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
-
-      ClusterStatus clusterStatus = mock(ClusterStatus.class);
-      when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
-      when(clusterStatus.getLoad(sn)).thenReturn(sl);
-
-      loadBalancer.setClusterStatus(clusterStatus);
-    }
-    assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
-    assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
-
-    Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
-    int i = 0;
-    while(loads.size() > 0) {
-      RegionLoad rl = loads.remove();
-      assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
-      i ++;
-    }
-  }
-
-  /**
-   * Test the load balancing algorithm.
-   *
-   * Invariant is that all servers should be hosting either floor(average) or
-   * ceiling(average)
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testBalanceCluster() throws Exception {
-
-    for (int[] mockCluster : clusterStateMocks) {
-      Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
-      List<ServerAndLoad> list = convertToList(servers);
-      LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
-      List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
-      List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
-      LOG.info("Mock Balance : " + printMock(balancedCluster));
-      assertClusterAsBalanced(balancedCluster);
-      List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(servers);
-      assertNull(secondPlans);
-      for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
-        returnRegions(entry.getValue());
-        returnServer(entry.getKey());
-      }
-    }
-
-  }
-
-  @Test
-  public void testMoveCost() throws Exception {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.MoveCostFunction(conf);
-    for (int[] mockCluster : clusterStateMocks) {
-      BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
-      costFunction.init(cluster);
-      double cost = costFunction.cost();
-      assertEquals(0.0f, cost, 0.001);
-
-      // cluster region number is smaller than maxMoves=600
-      cluster.setNumRegions(200);
-      cluster.setNumMovedRegions(10);
-      cost = costFunction.cost();
-      assertEquals(0.05f, cost, 0.001);
-      cluster.setNumMovedRegions(100);
-      cost = costFunction.cost();
-      assertEquals(0.5f, cost, 0.001);
-      cluster.setNumMovedRegions(200);
-      cost = costFunction.cost();
-      assertEquals(1.0f, cost, 0.001);
-
-
-      // cluster region number is bigger than maxMoves=2500
-      cluster.setNumRegions(10000);
-      cluster.setNumMovedRegions(250);
-      cost = costFunction.cost();
-      assertEquals(0.1f, cost, 0.001);
-      cluster.setNumMovedRegions(1250);
-      cost = costFunction.cost();
-      assertEquals(0.5f, cost, 0.001);
-      cluster.setNumMovedRegions(2500);
-      cost = costFunction.cost();
-      assertEquals(1.0f, cost, 0.01);
-    }
-  }
-
-  @Test
-  public void testSkewCost() {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
-    for (int[] mockCluster : clusterStateMocks) {
-      costFunction.init(mockCluster(mockCluster));
-      double cost = costFunction.cost();
-      assertTrue(cost >= 0);
-      assertTrue(cost <= 1.01);
-    }
-
-    costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
-    assertEquals(0,costFunction.cost(), 0.01);
-    costFunction.init(mockCluster(new int[]{0, 0, 0, 1, 1}));
-    assertEquals(0, costFunction.cost(), 0.01);
-    costFunction.init(mockCluster(new int[]{0, 0, 1, 1, 1}));
-    assertEquals(0, costFunction.cost(), 0.01);
-    costFunction.init(mockCluster(new int[]{0, 1, 1, 1, 1}));
-    assertEquals(0, costFunction.cost(), 0.01);
-    costFunction.init(mockCluster(new int[]{1, 1, 1, 1, 1}));
-    assertEquals(0, costFunction.cost(), 0.01);
-    costFunction.init(mockCluster(new int[]{10000, 0, 0, 0, 0}));
-    assertEquals(1, costFunction.cost(), 0.01);
-  }
-
-  @Test
-  public void testTableSkewCost() {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
-    for (int[] mockCluster : clusterStateMocks) {
-      BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
-      costFunction.init(cluster);
-      double cost = costFunction.cost();
-      assertTrue(cost >= 0);
-      assertTrue(cost <= 1.01);
-    }
-  }
-
-  @Test
-  public void testCostFromArray() {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFromRegionLoadFunction
-        costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
-    costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
-
-    double[] statOne = new double[100];
-    for (int i =0; i < 100; i++) {
-      statOne[i] = 10;
-    }
-    assertEquals(0, costFunction.costFromArray(statOne), 0.01);
-
-    double[] statTwo= new double[101];
-    for (int i =0; i < 100; i++) {
-      statTwo[i] = 0;
-    }
-    statTwo[100] = 100;
-    assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
-
-    double[] statThree = new double[200];
-    for (int i =0; i < 100; i++) {
-      statThree[i] = (0);
-      statThree[i+100] = 100;
-    }
-    assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
-  }
-
-  @Test(timeout =  60000)
-  public void testLosingRs() throws Exception {
-    int numNodes = 3;
-    int numRegions = 20;
-    int numRegionsPerServer = 3; //all servers except one
-    int replication = 1;
-    int numTables = 2;
-
-    Map<ServerName, List<HRegionInfo>> serverMap =
-        createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
-    List<ServerAndLoad> list = convertToList(serverMap);
-
-
-    List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
-    assertNotNull(plans);
-
-    // Apply the plan to the mock cluster.
-    List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
-
-    assertClusterAsBalanced(balancedCluster);
-
-    ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
-
-    ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
-
-    serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
-
-    plans = loadBalancer.balanceCluster(serverMap);
-    assertNull(plans);
-  }
-
-  @Test
-  public void testReplicaCost() {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
-    for (int[] mockCluster : clusterStateMocks) {
-      BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
-      costFunction.init(cluster);
-      double cost = costFunction.cost();
-      assertTrue(cost >= 0);
-      assertTrue(cost <= 1.01);
-    }
-  }
-
-  @Test
-  public void testReplicaCostForReplicas() {
-    Configuration conf = HBaseConfiguration.create();
-    StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
-
-    int [] servers = new int[] {3,3,3,3,3};
-    TreeMap<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(servers);
-
-    BaseLoadBalancer.Cluster cluster;
-
-    cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
-    costFunction.init(cluster);
-    double costWithoutReplicas = costFunction.cost();
-    assertEquals(0, costWithoutReplicas, 0);
-
-    // replicate the region from first server to the last server
-    HRegionInfo replica1 = RegionReplicaUtil.getRegionInfoForReplica(
-      clusterState.firstEntry().getValue().get(0),1);
-    clusterState.lastEntry().getValue().add(replica1);
-
-    cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
-    costFunction.init(cluster);
-    double costWith1ReplicaDifferentServer = costFunction.cost();
-
-    assertEquals(0, costWith1ReplicaDifferentServer, 0);
-
-    // add a third replica to the last server
-    HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2);
-    clusterState.lastEntry().getValue().add(replica2);
-
-    cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
-    costFunction.init(cluster);
-    double costWith1ReplicaSameServer = costFunction.cost();
-
-    assertTrue(costWith1ReplicaDifferentServer < costWith1ReplicaSameServer);
-
-    // test with replication = 4 for following:
-
-    HRegionInfo replica3;
-    Iterator<Entry<ServerName, List<HRegionInfo>>> it;
-    Entry<ServerName, List<HRegionInfo>> entry;
-
-    clusterState = mockClusterServers(servers);
-    it = clusterState.entrySet().iterator();
-    entry = it.next(); //first server
-    HRegionInfo hri = entry.getValue().get(0);
-    replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
-    replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
-    replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
-    entry.getValue().add(replica1);
-    entry.getValue().add(replica2);
-    it.next().getValue().add(replica3); //2nd server
-
-    cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
-    costFunction.init(cluster);
-    double costWith3ReplicasSameServer = costFunction.cost();
-
-    clusterState = mockClusterServers(servers);
-    hri = clusterState.firstEntry().getValue().get(0);
-    replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
-    replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
-    replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
-
-    clusterState.firstEntry().getValue().add(replica1);
-    clusterState.lastEntry().getValue().add(replica2);
-    clusterState.lastEntry().getValue().add(replica3);
-
-    cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
-    costFunction.init(cluster);
-    double costWith2ReplicasOnTwoServers = costFunction.cost();
-
-    assertTrue(costWith2ReplicasOnTwoServers < costWith3ReplicasSameServer);
-  }
-
-  @Test
-  public void testNeedsBalanceForColocatedReplicas() {
-    // check for the case where there are two hosts and with one rack, and where
-    // both the replicas are hosted on the same server
-    List<HRegionInfo> regions = randomRegions(1);
-    ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
-    ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
-    Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
-    map.put(s1, regions);
-    regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
-    // until the step above s1 holds two replicas of a region
-    regions = randomRegions(1);
-    map.put(s2, regions);
-    assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
-    // check for the case where there are two hosts on the same rack and there are two racks
-    // and both the replicas are on the same rack
-    map.clear();
-    regions = randomRegions(1);
-    List<HRegionInfo> regionsOnS2 = new ArrayList<HRegionInfo>(1);
-    regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
-    map.put(s1, regions);
-    map.put(s2, regionsOnS2);
-    // add another server so that the cluster has some host on another rack
-    map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
-    assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null,
-        new ForTestRackManagerOne())));
-  }
-
-  @Test (timeout = 60000)
-  public void testSmallCluster() {
-    int numNodes = 10;
-    int numRegions = 1000;
-    int numRegionsPerServer = 40; //all servers except one
-    int replication = 1;
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 60000)
-  public void testSmallCluster2() {
-    int numNodes = 20;
-    int numRegions = 2000;
-    int numRegionsPerServer = 40; //all servers except one
-    int replication = 1;
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 60000)
-  public void testSmallCluster3() {
-    int numNodes = 20;
-    int numRegions = 2000;
-    int numRegionsPerServer = 1; // all servers except one
-    int replication = 1;
-    int numTables = 10;
-    /* fails because of max moves */
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false);
-  }
-
-  @Test (timeout = 800000)
-  public void testMidCluster() {
-    int numNodes = 100;
-    int numRegions = 10000;
-    int numRegionsPerServer = 60; // all servers except one
-    int replication = 1;
-    int numTables = 40;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testMidCluster2() {
-    int numNodes = 200;
-    int numRegions = 100000;
-    int numRegionsPerServer = 40; // all servers except one
-    int replication = 1;
-    int numTables = 400;
-    testWithCluster(numNodes,
-        numRegions,
-        numRegionsPerServer,
-        replication,
-        numTables,
-        false, /* num large num regions means may not always get to best balance with one run */
-        false);
-  }
-
-
-  @Test (timeout = 800000)
-  public void testMidCluster3() {
-    int numNodes = 100;
-    int numRegions = 2000;
-    int numRegionsPerServer = 9; // all servers except one
-    int replication = 1;
-    int numTables = 110;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-    // TODO(eclark): Make sure that the tables are well distributed.
-  }
-
-  @Test
-  public void testLargeCluster() {
-    int numNodes = 1000;
-    int numRegions = 100000; //100 regions per RS
-    int numRegionsPerServer = 80; //all servers except one
-    int numTables = 100;
-    int replication = 1;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicasOnSmallCluster() {
-    int numNodes = 10;
-    int numRegions = 1000;
-    int replication = 3; // 3 replicas per region
-    int numRegionsPerServer = 80; //all regions are mostly balanced
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicationOnMidClusterSameHosts() {
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    loadBalancer.setConf(conf);
-    int numHosts = 100;
-    int numRegions = 100 * 100;
-    int replication = 3; // 3 replicas per region
-    int numRegionsPerServer = 5;
-    int numTables = 10;
-    Map<ServerName, List<HRegionInfo>> serverMap =
-        createServerMap(numHosts, numRegions, numRegionsPerServer, replication, numTables);
-    int numNodesPerHost = 4;
-
-    // create a new map with 4 RS per host.
-    Map<ServerName, List<HRegionInfo>> newServerMap = new TreeMap<ServerName, List<HRegionInfo>>(serverMap);
-    for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverMap.entrySet()) {
-      for (int i=1; i < numNodesPerHost; i++) {
-        ServerName s1 = entry.getKey();
-        ServerName s2 = ServerName.valueOf(s1.getHostname(), s1.getPort() + i, 1); // create an RS for the same host
-        newServerMap.put(s2, new ArrayList<HRegionInfo>());
-      }
-    }
-
-    testWithCluster(newServerMap, null, true, true);
-  }
-
-  private static class ForTestRackManager extends RackManager {
-    int numRacks;
-    public ForTestRackManager(int numRacks) {
-      this.numRacks = numRacks;
-    }
-    @Override
-    public String getRack(ServerName server) {
-      return "rack_" + (server.hashCode() % numRacks);
-    }
-  }
-
-  private static class ForTestRackManagerOne extends RackManager {
-  @Override
-    public String getRack(ServerName server) {
-      return server.getHostname().endsWith("1") ? "rack1" : "rack2";
-    }
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicationOnMidClusterWithRacks() {
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
-    loadBalancer.setConf(conf);
-    int numNodes = 30;
-    int numRegions = numNodes * 30;
-    int replication = 3; // 3 replicas per region
-    int numRegionsPerServer = 28;
-    int numTables = 10;
-    int numRacks = 4; // all replicas should be on a different rack
-    Map<ServerName, List<HRegionInfo>> serverMap =
-        createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
-    RackManager rm = new ForTestRackManager(numRacks);
-
-    testWithCluster(serverMap, rm, false, true);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
deleted file mode 100644
index 5008ac5..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.balancer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.testclassification.FlakeyTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({FlakeyTests.class, MediumTests.class})
-public class TestStochasticLoadBalancer2 extends BalancerTestBase {
-  private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer2.class);
-
-  @Test (timeout = 800000)
-  public void testRegionReplicasOnMidCluster() {
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
-    conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    TestStochasticLoadBalancer.loadBalancer.setConf(conf);
-    int numNodes = 200;
-    int numRegions = 40 * 200;
-    int replication = 3; // 3 replicas per region
-    int numRegionsPerServer = 30; //all regions are mostly balanced
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicasOnLargeCluster() {
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
-    conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    loadBalancer.setConf(conf);
-    int numNodes = 1000;
-    int numRegions = 20 * numNodes; // 20 * replication regions per RS
-    int numRegionsPerServer = 19; // all servers except one
-    int numTables = 100;
-    int replication = 3;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicasOnMidClusterHighReplication() {
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
-    conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    loadBalancer.setConf(conf);
-    int numNodes = 80;
-    int numRegions = 6 * numNodes;
-    int replication = 80; // 80 replicas per region, one for each server
-    int numRegionsPerServer = 5;
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true);
-  }
-
-  @Test (timeout = 800000)
-  public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() {
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
-    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
-    conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
-    loadBalancer.setConf(conf);
-    int numNodes = 40;
-    int numRegions = 6 * 50;
-    int replication = 50; // 50 replicas per region, more than numNodes
-    int numRegionsPerServer = 6;
-    int numTables = 10;
-    testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
deleted file mode 100644
index c8d3a62..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
+++ /dev/null
@@ -1,514 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.master.procedure;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.procedure2.Procedure;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
-import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure;
-import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
-import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateTableState;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DeleteTableState;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DisableTableState;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.EnableTableState;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.TruncateTableState;
-import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ModifyRegionUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-@Category({MasterTests.class, LargeTests.class})
-public class TestMasterFailoverWithProcedures {
-  private static final Log LOG = LogFactory.getLog(TestMasterFailoverWithProcedures.class);
-
-  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
-  private static void setupConf(Configuration conf) {
-    // don't waste time retrying with the roll, the test is already slow enough.
-    conf.setInt("hbase.procedure.store.wal.max.retries.before.roll", 1);
-    conf.setInt("hbase.procedure.store.wal.wait.before.roll", 0);
-    conf.setInt("hbase.procedure.store.wal.max.roll.retries", 1);
-    conf.setInt("hbase.procedure.store.wal.sync.failure.roll.max", 1);
-  }
-
-  @Before
-  public void setup() throws Exception {
-    setupConf(UTIL.getConfiguration());
-    UTIL.startMiniCluster(2, 1);
-
-    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false);
-    ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    try {
-      UTIL.shutdownMiniCluster();
-    } catch (Exception e) {
-      LOG.warn("failure shutting down cluster", e);
-    }
-  }
-
-  @Test(timeout=60000)
-  public void testWalRecoverLease() throws Exception {
-    final ProcedureStore masterStore = getMasterProcedureExecutor().getStore();
-    assertTrue("expected WALStore for this test", masterStore instanceof WALProcedureStore);
-
-    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
-    // Abort Latch for the master store
-    final CountDownLatch masterStoreAbort = new CountDownLatch(1);
-    masterStore.registerListener(new ProcedureStore.ProcedureStoreListener() {
-      @Override
-      public void postSync() {}
-
-      @Override
-      public void abortProcess() {
-        LOG.debug("Abort store of Master");
-        masterStoreAbort.countDown();
-      }
-    });
-
-    // startup a fake master the new WAL store will take the lease
-    // and the active master should abort.
-    HMaster backupMaster3 = Mockito.mock(HMaster.class);
-    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
-    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
-    final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(),
-        firstMaster.getMasterFileSystem().getFileSystem(),
-        ((WALProcedureStore)masterStore).getLogDir(),
-        new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
-    // Abort Latch for the test store
-    final CountDownLatch backupStore3Abort = new CountDownLatch(1);
-    backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() {
-      @Override
-      public void postSync() {}
-
-      @Override
-      public void abortProcess() {
-        LOG.debug("Abort store of backupMaster3");
-        backupStore3Abort.countDown();
-        backupStore3.stop(true);
-      }
-    });
-    backupStore3.start(1);
-    backupStore3.recoverLease();
-
-    // Try to trigger a command on the master (WAL lease expired on the active one)
-    HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(TableName.valueOf("mtb"), "f");
-    HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, null);
-    LOG.debug("submit proc");
-    try {
-      getMasterProcedureExecutor().submitProcedure(
-        new CreateTableProcedure(getMasterProcedureExecutor().getEnvironment(), htd, regions));
-      fail("expected RuntimeException 'sync aborted'");
-    } catch (RuntimeException e) {
-      LOG.info("got " + e.getMessage());
-    }
-    LOG.debug("wait master store abort");
-    masterStoreAbort.await();
-
-    // Now the real backup master should start up
-    LOG.debug("wait backup master to startup");
-    waitBackupMaster(UTIL, firstMaster);
-    assertEquals(true, firstMaster.isStopped());
-
-    // wait the store in here to abort (the test will fail due to timeout if it doesn't)
-    LOG.debug("wait the store to abort");
-    backupStore3.getStoreTracker().setDeleted(1, false);
-    try {
-      backupStore3.delete(1);
-      fail("expected RuntimeException 'sync aborted'");
-    } catch (RuntimeException e) {
-      LOG.info("got " + e.getMessage());
-    }
-    backupStore3Abort.await();
-  }
-
-  /**
-   * Tests proper fencing in case the current WAL store is fenced
-   */
-  @Test
-  public void testWALfencingWithoutWALRolling() throws IOException {
-    testWALfencing(false);
-  }
-
-  /**
-   * Tests proper fencing in case the current WAL store does not receive writes until after the
-   * new WAL does a couple of WAL rolls.
-   */
-  @Test
-  public void testWALfencingWithWALRolling() throws IOException {
-    testWALfencing(true);
-  }
-
-  public void testWALfencing(boolean walRolls) throws IOException {
-    final ProcedureStore procStore = getMasterProcedureExecutor().getStore();
-    assertTrue("expected WALStore for this test", procStore instanceof WALProcedureStore);
-
-    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
-
-    // cause WAL rolling after a delete in WAL:
-    firstMaster.getConfiguration().setLong("hbase.procedure.store.wal.roll.threshold", 1);
-
-    HMaster backupMaster3 = Mockito.mock(HMaster.class);
-    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
-    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
-    final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(),
-        firstMaster.getMasterFileSystem().getFileSystem(),
-        ((WALProcedureStore)procStore).getLogDir(),
-        new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
-
-    // start a second store which should fence the first one out
-    LOG.info("Starting new WALProcedureStore");
-    procStore2.start(1);
-    procStore2.recoverLease();
-
-    // before writing back to the WAL store, optionally do a couple of WAL rolls (which causes
-    // to delete the old WAL files).
-    if (walRolls) {
-      LOG.info("Inserting into second WALProcedureStore, causing WAL rolls");
-      for (int i = 0; i < 512; i++) {
-        // insert something to the second store then delete it, causing a WAL roll(s)
-        Procedure proc2 = new TestProcedure(i);
-        procStore2.insert(proc2, null);
-        procStore2.delete(proc2.getProcId()); // delete the procedure so that the WAL is removed later
-      }
-    }
-
-    // Now, insert something to the first store, should fail.
-    // If the store does a WAL roll and continue with another logId without checking higher logIds
-    // it will incorrectly succeed.
-    LOG.info("Inserting into first WALProcedureStore");
-    try {
-      procStore.insert(new TestProcedure(11), null);
-      fail("Inserting into Procedure Store should have failed");
-    } catch (Exception ex) {
-      LOG.info("Received expected exception", ex);
-    }
-  }
-
-  // ==========================================================================
-  //  Test Create Table
-  // ==========================================================================
-  @Test(timeout=60000)
-  public void testCreateWithFailover() throws Exception {
-    // TODO: Should we try every step? (master failover takes long time)
-    // It is already covered by TestCreateTableProcedure
-    // but without the master restart, only the executor/store is restarted.
-    // Without Master restart we may not find bug in the procedure code
-    // like missing "wait" for resources to be available (e.g. RS)
-    testCreateWithFailoverAtStep(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS.ordinal());
-  }
-
-  private void testCreateWithFailoverAtStep(final int step) throws Exception {
-    final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step);
-
-    // create the table
-    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
-    ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
-
-    // Start the Create procedure && kill the executor
-    byte[][] splitKeys = null;
-    HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
-    HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
-    long procId = procExec.submitProcedure(
-      new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
-    testRecoveryAndDoubleExecution(UTIL, procId, step, CreateTableState.values());
-
-    MasterProcedureTestingUtility.validateTableCreation(
-      UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
-  }
-
-  // ==========================================================================
-  //  Test Delete Table
-  // ==========================================================================
-  @Test(timeout=60000)
-  public void testDeleteWithFailover() throws Exception {
-    // TODO: Should we try every step? (master failover takes long time)
-    // It is already covered by TestDeleteTableProcedure
-    // but without the master restart, only the executor/store is restarted.
-    // Without Master restart we may not find bug in the procedure code
-    // like missing "wait" for resources to be available (e.g. RS)
-    testDeleteWithFailoverAtStep(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS.ordinal());
-  }
-
-  private void testDeleteWithFailoverAtStep(final int step) throws Exception {
-    final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step);
-
-    // create the table
-    byte[][] splitKeys = null;
-    HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
-      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
-    Path tableDir = FSUtils.getTableDir(getRootDir(), tableName);
-    MasterProcedureTestingUtility.validateTableCreation(
-      UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
-    UTIL.getHBaseAdmin().disableTable(tableName);
-
-    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
-    ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
-
-    // Start the Delete procedure && kill the executor
-    long procId = procExec.submitProcedure(
-      new DeleteTableProcedure(procExec.getEnvironment(), tableName));
-    testRecoveryAndDoubleExecution(UTIL, procId, step, DeleteTableState.values());
-
-    MasterProcedureTestingUtility.validateTableDeletion(
-      UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
-  }
-
-  // ==========================================================================
-  //  Test Truncate Table
-  // ==========================================================================
-  @Test(timeout=90000)
-  public void testTruncateWithFailover() throws Exception {
-    // TODO: Should we try every step? (master failover takes long time)
-    // It is already covered by TestTruncateTableProcedure
-    // but without the master restart, only the executor/store is restarted.
-    // Without Master restart we may not find bug in the procedure code
-    // like missing "wait" for resources to be available (e.g. RS)
-    testTruncateWithFailoverAtStep(true, TruncateTableState.TRUNCATE_TABLE_ADD_TO_META.ordinal());
-  }
-
-  private void testTruncateWithFailoverAtStep(final boolean preserveSplits, final int step)
-      throws Exception {
-    final TableName tableName = TableName.valueOf("testTruncateWithFailoverAtStep" + step);
-
-    // create the table
-    final String[] families = new String[] { "f1", "f2" };
-    final byte[][] splitKeys = new byte[][] {
-      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
-    };
-    HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
-      getMasterProcedureExecutor(), tableName, splitKeys, families);
-    // load and verify that there are rows in the table
-    MasterProcedureTestingUtility.loadData(
-      UTIL.getConnection(), tableName, 100, splitKeys, families);
-    assertEquals(100, UTIL.countRows(tableName));
-    // disable the table
-    UTIL.getHBaseAdmin().disableTable(tableName);
-
-    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
-    // Start the Truncate procedure && kill the executor
-    long procId = procExec.submitProcedure(
-      new TruncateTableProcedure(procExec.getEnvironment(), tableName, preserveSplits));
-    testRecoveryAndDoubleExecution(UTIL, procId, step, TruncateTableState.values());
-
-    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
-    UTIL.waitUntilAllRegionsAssigned(tableName);
-
-    // validate the table regions and layout
-    if (preserveSplits) {
-      assertEquals(1 + splitKeys.length, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
-    } else {
-      regions = UTIL.getHBaseAdmin().getTableRegions(tableName).toArray(new HRegionInfo[1]);
-      assertEquals(1, regions.length);
-    }
-    MasterProcedureTestingUtility.validateTableCreation(
-      UTIL.getHBaseCluster().getMaster(), tableName, regions, families);
-
-    // verify that there are no rows in the table
-    assertEquals(0, UTIL.countRows(tableName));
-
-    // verify that the table is read/writable
-    MasterProcedureTestingUtility.loadData(
-      UTIL.getConnection(), tableName, 50, splitKeys, families);
-    assertEquals(50, UTIL.countRows(tableName));
-  }
-
-  // ==========================================================================
-  //  Test Disable Table
-  // ==========================================================================
-  @Test(timeout=60000)
-  public void testDisableTableWithFailover() throws Exception {
-    // TODO: Should we try every step? (master failover takes long time)
-    // It is already covered by TestDisableTableProcedure
-    // but without the master restart, only the executor/store is restarted.
-    // Without Master restart we may not find bug in the procedure code
-    // like missing "wait" for resources to be available (e.g. RS)
-    testDisableTableWithFailoverAtStep(
-      DisableTableState.DISABLE_TABLE_MARK_REGIONS_OFFLINE.ordinal());
-  }
-
-  private void testDisableTableWithFailoverAtStep(final int step) throws Exception {
-    final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step);
-
-    // create the table
-    final byte[][] splitKeys = new byte[][] {
-      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
-    };
-    MasterProcedureTestingUtility.createTable(
-      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
-
-    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
-    // Start the Delete procedure && kill the executor
-    long procId = procExec.submitProcedure(
-      new DisableTableProcedure(procExec.getEnvironment(), tableName, false));
-    testRecoveryAndDoubleExecution(UTIL, procId, step, DisableTableState.values());
-
-    MasterProcedureTestingUtility.validateTableIsDisabled(
-      UTIL.getHBaseCluster().getMaster(), tableName);
-  }
-
-  // ==========================================================================
-  //  Test Enable Table
-  // ==========================================================================
-  @Test(timeout=60000)
-  public void testEnableTableWithFailover() throws Exception {
-    // TODO: Should we try every step? (master failover takes long time)
-    // It is already covered by TestEnableTableProcedure
-    // but without the master restart, only the executor/store is restarted.
-    // Without Master restart we may not find bug in the procedure code
-    // like missing "wait" for resources to be available (e.g. RS)
-    testEnableTableWithFailoverAtStep(
-      EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE.ordinal());
-  }
-
-  private void testEnableTableWithFailoverAtStep(final int step) throws Exception {
-    final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step);
-
-    // create the table
-    final byte[][] splitKeys = new byte[][] {
-      Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
-    };
-    MasterProcedureTestingUtility.createTable(
-      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
-    UTIL.getHBaseAdmin().disableTable(tableName);
-
-    ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
-    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
-    // Start the Delete procedure && kill the executor
-    long procId = procExec.submitProcedure(
-      new EnableTableProcedure(procExec.getEnvironment(), tableName, false));
-    testRecoveryAndDoubleExecution(UTIL, procId, step, EnableTableState.values());
-
-    MasterProcedureTestingUtility.validateTableIsEnabled(
-      UTIL.getHBaseCluster().getMaster(), tableName);
-  }
-
-  // ==========================================================================
-  //  Test Helpers
-  // ==========================================================================
-  public static <TState> void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil,
-      final long procId, final int lastStepBeforeFailover, TState[] states) throws Exception {
-    ProcedureExecutor<MasterProcedureEnv> procExec =
-      testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
-    ProcedureTestingUtility.waitProcedure(procExec, procId);
-
-    for (int i = 0; i < lastStepBeforeFailover; ++i) {
-      LOG.info("Restart "+ i +" exec state: " + states[i]);
-      ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
-      ProcedureTestingUtility.restart(procExec);
-      ProcedureTestingUtility.waitProcedure(procExec, procId);
-    }
-    ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
-
-    LOG.info("Trigger master failover");
-    masterFailover(testUtil);
-
-    procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
-    ProcedureTestingUtility.waitProcedure(procExec, procId);
-    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
-  }
-
-  // ==========================================================================
-  //  Master failover utils
-  // ==========================================================================
-  public static void masterFailover(final HBaseTestingUtility testUtil)
-      throws Exception {
-    MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
-
-    // Kill the master
-    HMaster oldMaster = cluster.getMaster();
-    cluster.killMaster(cluster.getMaster().getServerName());
-
-    // Wait the secondary
-    waitBackupMaster(testUtil, oldMaster);
-  }
-
-  public static void waitBackupMaster(final HBaseTestingUtility testUtil,
-      final HMaster oldMaster) throws Exception {
-    MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
-
-    HMaster newMaster = cluster.getMaster();
-    while (newMaster == null || newMaster == oldMaster) {
-      Thread.sleep(250);
-      newMaster = cluster.getMaster();
-    }
-
-    while (!(newMaster.isActiveMaster() && newMaster.isInitialized())) {
-      Thread.sleep(250);
-    }
-  }
-
-  // ==========================================================================
-  //  Helpers
-  // ==========================================================================
-  private MasterProcedureEnv getMasterProcedureEnv() {
-    return getMasterProcedureExecutor().getEnvironment();
-  }
-
-  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
-    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
-  }
-
-  private FileSystem getFileSystem() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
-  }
-
-  private Path getRootDir() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
-  }
-
-  private Path getTempDir() {
-    return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getTempDir();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java
deleted file mode 100644
index 8379576..0000000
--- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.jruby.embed.PathType;
-import org.junit.Test;
-import org.junit.Ignore;
-import org.junit.experimental.categories.Category;
-
-@Category({ ClientTests.class, LargeTests.class })
-public class TestReplicationShell extends AbstractTestShell {
-  @Ignore ("Disabled because hangs on occasion.. about 10% of the time") @Test
-  public void testRunShellTests() throws IOException {
-    System.setProperty("shell.test.include", "replication_admin_test.rb");
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/93023f54/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
deleted file mode 100644
index 976ba45..0000000
--- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.jruby.embed.PathType;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({ ClientTests.class, LargeTests.class })
-public class TestShell extends AbstractTestShell {
-
-  @Test
-  public void testRunShellTests() throws IOException {
-    System.setProperty("shell.test.exclude", "replication_admin_test.rb");
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
-  }
-
-}


Mime
View raw message