hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject hbase git commit: HBASE-14211 Add more rigorous integration tests of splits
Date Mon, 12 Oct 2015 20:25:32 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 a7bf4198b -> 801900956


HBASE-14211 Add more rigorous integration tests of splits

Summary:
Intgration tests don't currently have a lot of splits going on while there is IO.
This changes that by changing the split aglorithm and the max region hfile size. That should
make things split more.

Additionally this allows ITBLL to start with just one region if hbase.test.pre-split-table
is False.

Test Plan: Test on a cluster with ITBLL slow determinitic monkey and stress am monkey.

Differential Revision: https://reviews.facebook.net/D44181


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80190095
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80190095
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80190095

Branch: refs/heads/branch-1
Commit: 8019009563eff5fbe2cbaf91662d989087aa5f6d
Parents: a7bf419
Author: Elliott Clark <eclark@apache.org>
Authored: Tue Aug 11 16:52:54 2015 -0700
Committer: Elliott Clark <eclark@apache.org>
Committed: Mon Oct 12 13:25:10 2015 -0700

----------------------------------------------------------------------
 .../chaos/actions/ChangeSplitPolicyAction.java  | 58 ++++++++++++++
 .../actions/DecreaseMaxHFileSizeAction.java     | 79 +++++++++++++++++++
 .../actions/SplitAllRegionOfTableAction.java    | 41 ++++++++++
 .../hbase/chaos/factories/MonkeyConstants.java  | 83 ++++++++++----------
 .../SlowDeterministicMonkeyFactory.java         | 34 +++-----
 .../StressAssignmentManagerMonkeyFactory.java   | 48 +++++------
 .../test/IntegrationTestBigLinkedList.java      | 35 ++++++---
 .../master/balancer/StochasticLoadBalancer.java |  1 -
 .../hadoop/hbase/HBaseTestingUtility.java       |  5 +-
 9 files changed, 277 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java
new file mode 100644
index 0000000..b5f759f
--- /dev/null
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.chaos.actions;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy;
+
+import java.util.Random;
+
+public class ChangeSplitPolicyAction extends Action {
+  private final TableName tableName;
+  private final String[] possiblePolicies;
+  private final Random random;
+
+  public ChangeSplitPolicyAction(TableName tableName) {
+    this.tableName = tableName;
+    possiblePolicies = new String[] {
+        IncreasingToUpperBoundRegionSplitPolicy.class.getName(),
+        ConstantSizeRegionSplitPolicy.class.getName(),
+        DisabledRegionSplitPolicy.class.getName()
+    };
+    this.random = new Random();
+  }
+
+
+  @Override
+  public void perform() throws Exception {
+    HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
+    Admin admin = util.getHBaseAdmin();
+
+    LOG.info("Performing action: Change split policy of table " + tableName);
+    HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName);
+    String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)];
+    tableDescriptor.setRegionSplitPolicyClassName(chosenPolicy);
+    LOG.info("Changing "  + tableName + " split policy to " + chosenPolicy);
+    admin.modifyTable(tableName, tableDescriptor);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java
new file mode 100644
index 0000000..55a34f4
--- /dev/null
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.chaos.actions;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+
+import java.util.Random;
+
+public class DecreaseMaxHFileSizeAction extends Action {
+
+  private static final long minFileSize = 1 *  1024 * 1024 * 1024L;
+
+  private final long sleepTime;
+  private final TableName tableName;
+  private final Random random;
+
+  public DecreaseMaxHFileSizeAction(long sleepTime, TableName tableName) {
+    this.sleepTime = sleepTime;
+    this.tableName = tableName;
+    this.random = new Random();
+  }
+
+  @Override
+  public void perform() throws Exception {
+    HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
+    Admin admin = util.getHBaseAdmin();
+    HTableDescriptor htd = admin.getTableDescriptor(tableName);
+
+    // Try and get the current value.
+    long currentValue = htd.getMaxFileSize();
+
+    // If the current value is not set use the default for the cluster.
+    // If configs are really weird this might not work.
+    // That's ok. We're trying to cause chaos.
+    if (currentValue <= 0) {
+      currentValue =
+          context.getHBaseCluster().getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+              HConstants.DEFAULT_MAX_FILE_SIZE);
+    }
+
+    // Decrease by 10% at a time.
+    long newValue = (long) (currentValue * 0.9);
+
+    // We don't want to go too far below 1gb.
+    // So go to about 1gb +/- 512 on each side.
+    newValue = Math.max(minFileSize, newValue) - (512 - random.nextInt(1024));
+
+    // Change the table descriptor.
+    htd.setMaxFileSize(newValue);
+
+    // modify the table.
+    admin.modifyTable(tableName, htd);
+
+    // Sleep some time.
+    if (sleepTime > 0) {
+      Thread.sleep(sleepTime);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java
new file mode 100644
index 0000000..6e7d0f9
--- /dev/null
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.chaos.actions;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+
+
+public class SplitAllRegionOfTableAction extends Action {
+  private final TableName tableName;
+
+  public SplitAllRegionOfTableAction(TableName tableName) {
+    this.tableName = tableName;
+  }
+
+
+  @Override
+  public void perform() throws Exception {
+    HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
+    Admin admin = util.getHBaseAdmin();
+
+    LOG.info("Performing action: Split all regions of  " + tableName);
+    admin.split(tableName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
index 3333b26..49f3b55 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
@@ -19,46 +19,47 @@ package org.apache.hadoop.hbase.chaos.factories;
 
 public interface MonkeyConstants {
 
-  public static final String PERIODIC_ACTION1_PERIOD = "sdm.action1.period";
-  public static final String PERIODIC_ACTION2_PERIOD = "sdm.action2.period";
-  public static final String PERIODIC_ACTION4_PERIOD = "sdm.action4.period";
-  public static final String COMPOSITE_ACTION3_PERIOD = "sdm.action3.period";
-  public static final String MOVE_REGIONS_MAX_TIME = "move.regions.max.time";
-  public static final String MOVE_REGIONS_SLEEP_TIME = "move.regions.sleep.time";
-  public static final String MOVE_RANDOM_REGION_SLEEP_TIME = "move.randomregion.sleep.time";
-  public static final String RESTART_RANDOM_RS_SLEEP_TIME = "restart.random.rs.sleep.time";
-  public static final String BATCH_RESTART_RS_SLEEP_TIME = "batch.restart.rs.sleep.time";
-  public static final String BATCH_RESTART_RS_RATIO = "batch.restart.rs.ratio";
-  public static final String RESTART_ACTIVE_MASTER_SLEEP_TIME = "restart.active.master.sleep.time";
-  public static final String ROLLING_BATCH_RESTART_RS_SLEEP_TIME = "rolling.batch.restart.rs.sleep.time";
-  public static final String ROLLING_BATCH_RESTART_RS_RATIO = "rolling.batch.restart.rs.ratio";
-  public static final String RESTART_RS_HOLDING_META_SLEEP_TIME = "restart.rs.holding.meta.sleep.time";
-  public static final String COMPACT_TABLE_ACTION_RATIO = "compact.table.ratio";
-  public static final String COMPACT_RANDOM_REGION_RATIO = "compact.random.region.ratio";
-  public static final String UNBALANCE_CHAOS_EVERY_MS = "unbalance.chaos.period";
-  public static final String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period";
-  public static final String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period";
-  public static final String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period";
-
-  public static final long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000;
-  public static final long DEFAULT_PERIODIC_ACTION2_PERIOD = 90 * 1000;
-  public static final long DEFAULT_PERIODIC_ACTION4_PERIOD = 90 * 1000;
-  public static final long DEFAULT_COMPOSITE_ACTION3_PERIOD = 150 * 1000;
-  public static final long DEFAULT_MOVE_REGIONS_MAX_TIME = 10 * 60 * 1000;
-  public static final long DEFAULT_MOVE_REGIONS_SLEEP_TIME = 800;
-  public static final long DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME = 800;
-  public static final long DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME = 60000;
-  public static final long DEFAULT_BATCH_RESTART_RS_SLEEP_TIME = 5000;
-  public static final float DEFAULT_BATCH_RESTART_RS_RATIO = 0.5f;
-  public static final long DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME = 5000;
-  public static final long DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME = 5000;
-  public static final float DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO = 1.0f;
-  public static final long DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME = 35000;
-  public static final float DEFAULT_COMPACT_TABLE_ACTION_RATIO = 0.5f;
-  public static final float DEFAULT_COMPACT_RANDOM_REGION_RATIO = 0.6f;
-  public static final long DEFAULT_UNBALANCE_CHAOS_EVERY_MS = 65 * 1000;
-  public static final long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000;
-  public static final long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000;
-  public static final long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000;
+  String PERIODIC_ACTION1_PERIOD = "sdm.action1.period";
+  String PERIODIC_ACTION2_PERIOD = "sdm.action2.period";
+  String PERIODIC_ACTION4_PERIOD = "sdm.action4.period";
+  String COMPOSITE_ACTION3_PERIOD = "sdm.action3.period";
+  String MOVE_REGIONS_MAX_TIME = "move.regions.max.time";
+  String MOVE_REGIONS_SLEEP_TIME = "move.regions.sleep.time";
+  String MOVE_RANDOM_REGION_SLEEP_TIME = "move.randomregion.sleep.time";
+  String RESTART_RANDOM_RS_SLEEP_TIME = "restart.random.rs.sleep.time";
+  String BATCH_RESTART_RS_SLEEP_TIME = "batch.restart.rs.sleep.time";
+  String BATCH_RESTART_RS_RATIO = "batch.restart.rs.ratio";
+  String RESTART_ACTIVE_MASTER_SLEEP_TIME = "restart.active.master.sleep.time";
+  String ROLLING_BATCH_RESTART_RS_SLEEP_TIME = "rolling.batch.restart.rs.sleep.time";
+  String ROLLING_BATCH_RESTART_RS_RATIO = "rolling.batch.restart.rs.ratio";
+  String RESTART_RS_HOLDING_META_SLEEP_TIME = "restart.rs.holding.meta.sleep.time";
+  String COMPACT_TABLE_ACTION_RATIO = "compact.table.ratio";
+  String COMPACT_RANDOM_REGION_RATIO = "compact.random.region.ratio";
+  String UNBALANCE_CHAOS_EVERY_MS = "unbalance.chaos.period";
+  String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period";
+  String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period";
+  String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period";
+  String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time";
 
+  long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000;
+  long DEFAULT_PERIODIC_ACTION2_PERIOD = 90 * 1000;
+  long DEFAULT_PERIODIC_ACTION4_PERIOD = 90 * 1000;
+  long DEFAULT_COMPOSITE_ACTION3_PERIOD = 150 * 1000;
+  long DEFAULT_MOVE_REGIONS_MAX_TIME = 10 * 60 * 1000;
+  long DEFAULT_MOVE_REGIONS_SLEEP_TIME = 800;
+  long DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME = 800;
+  long DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME = 60000;
+  long DEFAULT_BATCH_RESTART_RS_SLEEP_TIME = 5000;
+  float DEFAULT_BATCH_RESTART_RS_RATIO = 0.5f;
+  long DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME = 5000;
+  long DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME = 5000;
+  float DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO = 1.0f;
+  long DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME = 35000;
+  float DEFAULT_COMPACT_TABLE_ACTION_RATIO = 0.5f;
+  float DEFAULT_COMPACT_RANDOM_REGION_RATIO = 0.6f;
+  long DEFAULT_UNBALANCE_CHAOS_EVERY_MS = 65 * 1000;
+  long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000;
+  long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000;
+  long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000;
+  long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java
index 6195737..7aa4a07 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java
@@ -18,28 +18,7 @@
 
 package org.apache.hadoop.hbase.chaos.factories;
 
-import org.apache.hadoop.hbase.chaos.actions.Action;
-import org.apache.hadoop.hbase.chaos.actions.AddColumnAction;
-import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction;
-import org.apache.hadoop.hbase.chaos.actions.ChangeCompressionAction;
-import org.apache.hadoop.hbase.chaos.actions.ChangeBloomFilterAction;
-import org.apache.hadoop.hbase.chaos.actions.ChangeEncodingAction;
-import org.apache.hadoop.hbase.chaos.actions.ChangeVersionsAction;
-import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.CompactTableAction;
-import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
-import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.FlushTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction;
-import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction;
-import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction;
-import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction;
-import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction;
-import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction;
-import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction;
+import org.apache.hadoop.hbase.chaos.actions.*;
 import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
 import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
 import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
@@ -64,6 +43,7 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
   private long restartRsHoldingMetaSleepTime;
   private float compactTableRatio;
   private float compactRandomRegionRatio;
+  private long decreaseHFileSizeSleepTime;
 
   @Override
   public ChaosMonkey build() {
@@ -92,7 +72,8 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
         new ChangeEncodingAction(tableName),
         new ChangeCompressionAction(tableName),
         new ChangeBloomFilterAction(tableName),
-        new ChangeVersionsAction(tableName)
+        new ChangeVersionsAction(tableName),
+        new ChangeSplitPolicyAction(tableName),
     };
 
     // Destructive actions to mess things around.
@@ -105,7 +86,9 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
         new RestartActiveMasterAction(restartActiveMasterSleepTime),
         new RollingBatchRestartRsAction(rollingBatchRestartRSSleepTime,
             rollingBatchRestartRSRatio),
-        new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime)
+        new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime),
+        new DecreaseMaxHFileSizeAction(decreaseHFileSizeSleepTime, tableName),
+        new SplitAllRegionOfTableAction(tableName),
     };
 
     // Action to log more info for debugging
@@ -169,5 +152,8 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
       compactRandomRegionRatio = Float.parseFloat(this.properties.getProperty(
         MonkeyConstants.COMPACT_RANDOM_REGION_RATIO,
         MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO + ""));
+    decreaseHFileSizeSleepTime = Long.parseLong(this.properties.getProperty(
+        MonkeyConstants.DECREASE_HFILE_SIZE_SLEEP_TIME,
+        MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME + ""));
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java
index befb2fa..03471ab 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,22 +17,7 @@
  */
 package org.apache.hadoop.hbase.chaos.factories;
 
-import org.apache.hadoop.hbase.chaos.actions.Action;
-import org.apache.hadoop.hbase.chaos.actions.AddColumnAction;
-import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction;
-import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.CompactTableAction;
-import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
-import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.FlushTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction;
-import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction;
-import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction;
-import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction;
-import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction;
-import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction;
+import org.apache.hadoop.hbase.chaos.actions.*;
 import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
 import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
 import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
@@ -45,28 +30,35 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory
{
 
     // Actions that could slow down region movement.
     // These could also get regions stuck if there are issues.
-    Action[] actions1 = new Action[] {
+    Action[] actions1 = new Action[]{
         new CompactTableAction(tableName, 0.5f),
         new CompactRandomRegionOfTableAction(tableName, 0.6f),
         new FlushTableAction(tableName),
         new FlushRandomRegionOfTableAction(tableName)
     };
 
-    Action[] actions2 = new Action[] {
+    Action[] actions2 = new Action[]{
         new SplitRandomRegionOfTableAction(tableName),
         new MergeRandomAdjacentRegionsOfTableAction(tableName),
         new AddColumnAction(tableName),
         new RemoveColumnAction(tableName, columnFamilies),
-        new MoveRegionsOfTableAction(800, 1600, tableName),
-        new MoveRandomRegionOfTableAction(800, tableName),
-        new RestartRandomRsAction(60000),
-        new BatchRestartRsAction(5000, 0.5f),
-        new RollingBatchRestartRsAction(5000, 1.0f),
-        new RestartRsHoldingMetaAction(35000)
+        new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME,
+            1600,
+            tableName),
+        new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME,
+            tableName),
+        new RestartRandomRsAction(MonkeyConstants.DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME),
+        new BatchRestartRsAction(MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
0.5f),
+        new RollingBatchRestartRsAction(MonkeyConstants.DEFAULT_BATCH_RESTART_RS_SLEEP_TIME,
1.0f),
+        new RestartRsHoldingMetaAction(MonkeyConstants.DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME),
+        new ChangeSplitPolicyAction(tableName),
+        new SplitAllRegionOfTableAction(tableName),
+        new DecreaseMaxHFileSizeAction(MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME,
+            tableName),
     };
 
     // Action to log more info for debugging
-    Action[] actions3 = new Action[] {
+    Action[] actions3 = new Action[]{
         new DumpClusterStatusAction()
     };
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index f3d8360..d23da64 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -525,20 +525,31 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase
{
           // Always add these families. Just skip writing to them when we do not test per
CF flush.
           htd.addFamily(new HColumnDescriptor(BIG_FAMILY_NAME));
           htd.addFamily(new HColumnDescriptor(TINY_FAMILY_NAME));
-          int numberOfServers = admin.getClusterStatus().getServers().size();
-          if (numberOfServers == 0) {
-            throw new IllegalStateException("No live regionservers");
-          }
-          int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY,
-                                HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER);
-          int totalNumberOfRegions = numberOfServers * regionsPerServer;
-          LOG.info("Number of live regionservers: " + numberOfServers + ", " +
-              "pre-splitting table into " + totalNumberOfRegions + " regions " +
-              "(default regions per server: " + regionsPerServer + ")");
 
-          byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions);
+          // If we want to pre-split compute how many splits.
+          if (conf.getBoolean(HBaseTestingUtility.PRESPLIT_TEST_TABLE_KEY,
+              HBaseTestingUtility.PRESPLIT_TEST_TABLE)) {
+            int numberOfServers = admin.getClusterStatus().getServers().size();
+            if (numberOfServers == 0) {
+              throw new IllegalStateException("No live regionservers");
+            }
+            int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY,
+                HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER);
+            int totalNumberOfRegions = numberOfServers * regionsPerServer;
+            LOG.info("Number of live regionservers: " + numberOfServers + ", " +
+                "pre-splitting table into " + totalNumberOfRegions + " regions " +
+                "(default regions per server: " + regionsPerServer + ")");
+
 
-          admin.createTable(htd, splits);
+            byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions);
+
+            admin.createTable(htd, splits);
+          } else {
+            // Looks like we're just letting things play out.
+            // Create a table with on region by default.
+            // This will make the splitting work hard.
+            admin.createTable(htd);
+          }
         }
       } catch (MasterNotRunningException e) {
         LOG.error("Master not running", e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 4282a5e..e6326e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.ClusterStatus;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.RegionLoad;
 import org.apache.hadoop.hbase.ServerLoad;
 import org.apache.hadoop.hbase.ServerName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index bc730b8..a26a3bc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -155,8 +155,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
    * The default number of regions per regionserver when creating a pre-split
    * table.
    */
-  public static final int DEFAULT_REGIONS_PER_SERVER = 5;
+  public static final int DEFAULT_REGIONS_PER_SERVER = 3;
 
+
+  public static final String PRESPLIT_TEST_TABLE_KEY = "hbase.test.pre-split-table";
+  public static final boolean PRESPLIT_TEST_TABLE = true;
   /**
    * Set if we were passed a zkCluster.  If so, we won't shutdown zk as
    * part of general shutdown.


Mime
View raw message