Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 21B4B18F61 for ; Mon, 12 Oct 2015 20:25:33 +0000 (UTC) Received: (qmail 47249 invoked by uid 500); 12 Oct 2015 20:25:33 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 47207 invoked by uid 500); 12 Oct 2015 20:25:32 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 47197 invoked by uid 99); 12 Oct 2015 20:25:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 12 Oct 2015 20:25:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BE2E6E03E8; Mon, 12 Oct 2015 20:25:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: eclark@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-14211 Add more rigorous integration tests of splits Date: Mon, 12 Oct 2015 20:25:32 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1 a7bf4198b -> 801900956 HBASE-14211 Add more rigorous integration tests of splits Summary: Intgration tests don't currently have a lot of splits going on while there is IO. This changes that by changing the split aglorithm and the max region hfile size. That should make things split more. Additionally this allows ITBLL to start with just one region if hbase.test.pre-split-table is False. Test Plan: Test on a cluster with ITBLL slow determinitic monkey and stress am monkey. Differential Revision: https://reviews.facebook.net/D44181 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80190095 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80190095 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80190095 Branch: refs/heads/branch-1 Commit: 8019009563eff5fbe2cbaf91662d989087aa5f6d Parents: a7bf419 Author: Elliott Clark Authored: Tue Aug 11 16:52:54 2015 -0700 Committer: Elliott Clark Committed: Mon Oct 12 13:25:10 2015 -0700 ---------------------------------------------------------------------- .../chaos/actions/ChangeSplitPolicyAction.java | 58 ++++++++++++++ .../actions/DecreaseMaxHFileSizeAction.java | 79 +++++++++++++++++++ .../actions/SplitAllRegionOfTableAction.java | 41 ++++++++++ .../hbase/chaos/factories/MonkeyConstants.java | 83 ++++++++++---------- .../SlowDeterministicMonkeyFactory.java | 34 +++----- .../StressAssignmentManagerMonkeyFactory.java | 48 +++++------ .../test/IntegrationTestBigLinkedList.java | 35 ++++++--- .../master/balancer/StochasticLoadBalancer.java | 1 - .../hadoop/hbase/HBaseTestingUtility.java | 5 +- 9 files changed, 277 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java new file mode 100644 index 0000000..b5f759f --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.chaos.actions; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy; + +import java.util.Random; + +public class ChangeSplitPolicyAction extends Action { + private final TableName tableName; + private final String[] possiblePolicies; + private final Random random; + + public ChangeSplitPolicyAction(TableName tableName) { + this.tableName = tableName; + possiblePolicies = new String[] { + IncreasingToUpperBoundRegionSplitPolicy.class.getName(), + ConstantSizeRegionSplitPolicy.class.getName(), + DisabledRegionSplitPolicy.class.getName() + }; + this.random = new Random(); + } + + + @Override + public void perform() throws Exception { + HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); + Admin admin = util.getHBaseAdmin(); + + LOG.info("Performing action: Change split policy of table " + tableName); + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)]; + tableDescriptor.setRegionSplitPolicyClassName(chosenPolicy); + LOG.info("Changing " + tableName + " split policy to " + chosenPolicy); + admin.modifyTable(tableName, tableDescriptor); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java new file mode 100644 index 0000000..55a34f4 --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.actions; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; + +import java.util.Random; + +public class DecreaseMaxHFileSizeAction extends Action { + + private static final long minFileSize = 1 * 1024 * 1024 * 1024L; + + private final long sleepTime; + private final TableName tableName; + private final Random random; + + public DecreaseMaxHFileSizeAction(long sleepTime, TableName tableName) { + this.sleepTime = sleepTime; + this.tableName = tableName; + this.random = new Random(); + } + + @Override + public void perform() throws Exception { + HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); + Admin admin = util.getHBaseAdmin(); + HTableDescriptor htd = admin.getTableDescriptor(tableName); + + // Try and get the current value. + long currentValue = htd.getMaxFileSize(); + + // If the current value is not set use the default for the cluster. + // If configs are really weird this might not work. + // That's ok. We're trying to cause chaos. + if (currentValue <= 0) { + currentValue = + context.getHBaseCluster().getConf().getLong(HConstants.HREGION_MAX_FILESIZE, + HConstants.DEFAULT_MAX_FILE_SIZE); + } + + // Decrease by 10% at a time. + long newValue = (long) (currentValue * 0.9); + + // We don't want to go too far below 1gb. + // So go to about 1gb +/- 512 on each side. + newValue = Math.max(minFileSize, newValue) - (512 - random.nextInt(1024)); + + // Change the table descriptor. + htd.setMaxFileSize(newValue); + + // modify the table. + admin.modifyTable(tableName, htd); + + // Sleep some time. + if (sleepTime > 0) { + Thread.sleep(sleepTime); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java new file mode 100644 index 0000000..6e7d0f9 --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.chaos.actions; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; + + +public class SplitAllRegionOfTableAction extends Action { + private final TableName tableName; + + public SplitAllRegionOfTableAction(TableName tableName) { + this.tableName = tableName; + } + + + @Override + public void perform() throws Exception { + HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); + Admin admin = util.getHBaseAdmin(); + + LOG.info("Performing action: Split all regions of " + tableName); + admin.split(tableName); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java index 3333b26..49f3b55 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java @@ -19,46 +19,47 @@ package org.apache.hadoop.hbase.chaos.factories; public interface MonkeyConstants { - public static final String PERIODIC_ACTION1_PERIOD = "sdm.action1.period"; - public static final String PERIODIC_ACTION2_PERIOD = "sdm.action2.period"; - public static final String PERIODIC_ACTION4_PERIOD = "sdm.action4.period"; - public static final String COMPOSITE_ACTION3_PERIOD = "sdm.action3.period"; - public static final String MOVE_REGIONS_MAX_TIME = "move.regions.max.time"; - public static final String MOVE_REGIONS_SLEEP_TIME = "move.regions.sleep.time"; - public static final String MOVE_RANDOM_REGION_SLEEP_TIME = "move.randomregion.sleep.time"; - public static final String RESTART_RANDOM_RS_SLEEP_TIME = "restart.random.rs.sleep.time"; - public static final String BATCH_RESTART_RS_SLEEP_TIME = "batch.restart.rs.sleep.time"; - public static final String BATCH_RESTART_RS_RATIO = "batch.restart.rs.ratio"; - public static final String RESTART_ACTIVE_MASTER_SLEEP_TIME = "restart.active.master.sleep.time"; - public static final String ROLLING_BATCH_RESTART_RS_SLEEP_TIME = "rolling.batch.restart.rs.sleep.time"; - public static final String ROLLING_BATCH_RESTART_RS_RATIO = "rolling.batch.restart.rs.ratio"; - public static final String RESTART_RS_HOLDING_META_SLEEP_TIME = "restart.rs.holding.meta.sleep.time"; - public static final String COMPACT_TABLE_ACTION_RATIO = "compact.table.ratio"; - public static final String COMPACT_RANDOM_REGION_RATIO = "compact.random.region.ratio"; - public static final String UNBALANCE_CHAOS_EVERY_MS = "unbalance.chaos.period"; - public static final String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period"; - public static final String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period"; - public static final String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period"; - - public static final long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000; - public static final long DEFAULT_PERIODIC_ACTION2_PERIOD = 90 * 1000; - public static final long DEFAULT_PERIODIC_ACTION4_PERIOD = 90 * 1000; - public static final long DEFAULT_COMPOSITE_ACTION3_PERIOD = 150 * 1000; - public static final long DEFAULT_MOVE_REGIONS_MAX_TIME = 10 * 60 * 1000; - public static final long DEFAULT_MOVE_REGIONS_SLEEP_TIME = 800; - public static final long DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME = 800; - public static final long DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME = 60000; - public static final long DEFAULT_BATCH_RESTART_RS_SLEEP_TIME = 5000; - public static final float DEFAULT_BATCH_RESTART_RS_RATIO = 0.5f; - public static final long DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME = 5000; - public static final long DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME = 5000; - public static final float DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO = 1.0f; - public static final long DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME = 35000; - public static final float DEFAULT_COMPACT_TABLE_ACTION_RATIO = 0.5f; - public static final float DEFAULT_COMPACT_RANDOM_REGION_RATIO = 0.6f; - public static final long DEFAULT_UNBALANCE_CHAOS_EVERY_MS = 65 * 1000; - public static final long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000; - public static final long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000; - public static final long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000; + String PERIODIC_ACTION1_PERIOD = "sdm.action1.period"; + String PERIODIC_ACTION2_PERIOD = "sdm.action2.period"; + String PERIODIC_ACTION4_PERIOD = "sdm.action4.period"; + String COMPOSITE_ACTION3_PERIOD = "sdm.action3.period"; + String MOVE_REGIONS_MAX_TIME = "move.regions.max.time"; + String MOVE_REGIONS_SLEEP_TIME = "move.regions.sleep.time"; + String MOVE_RANDOM_REGION_SLEEP_TIME = "move.randomregion.sleep.time"; + String RESTART_RANDOM_RS_SLEEP_TIME = "restart.random.rs.sleep.time"; + String BATCH_RESTART_RS_SLEEP_TIME = "batch.restart.rs.sleep.time"; + String BATCH_RESTART_RS_RATIO = "batch.restart.rs.ratio"; + String RESTART_ACTIVE_MASTER_SLEEP_TIME = "restart.active.master.sleep.time"; + String ROLLING_BATCH_RESTART_RS_SLEEP_TIME = "rolling.batch.restart.rs.sleep.time"; + String ROLLING_BATCH_RESTART_RS_RATIO = "rolling.batch.restart.rs.ratio"; + String RESTART_RS_HOLDING_META_SLEEP_TIME = "restart.rs.holding.meta.sleep.time"; + String COMPACT_TABLE_ACTION_RATIO = "compact.table.ratio"; + String COMPACT_RANDOM_REGION_RATIO = "compact.random.region.ratio"; + String UNBALANCE_CHAOS_EVERY_MS = "unbalance.chaos.period"; + String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period"; + String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period"; + String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period"; + String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time"; + long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000; + long DEFAULT_PERIODIC_ACTION2_PERIOD = 90 * 1000; + long DEFAULT_PERIODIC_ACTION4_PERIOD = 90 * 1000; + long DEFAULT_COMPOSITE_ACTION3_PERIOD = 150 * 1000; + long DEFAULT_MOVE_REGIONS_MAX_TIME = 10 * 60 * 1000; + long DEFAULT_MOVE_REGIONS_SLEEP_TIME = 800; + long DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME = 800; + long DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME = 60000; + long DEFAULT_BATCH_RESTART_RS_SLEEP_TIME = 5000; + float DEFAULT_BATCH_RESTART_RS_RATIO = 0.5f; + long DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME = 5000; + long DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME = 5000; + float DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO = 1.0f; + long DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME = 35000; + float DEFAULT_COMPACT_TABLE_ACTION_RATIO = 0.5f; + float DEFAULT_COMPACT_RANDOM_REGION_RATIO = 0.6f; + long DEFAULT_UNBALANCE_CHAOS_EVERY_MS = 65 * 1000; + long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000; + long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000; + long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000; + long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000; } http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java index 6195737..7aa4a07 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java @@ -18,28 +18,7 @@ package org.apache.hadoop.hbase.chaos.factories; -import org.apache.hadoop.hbase.chaos.actions.Action; -import org.apache.hadoop.hbase.chaos.actions.AddColumnAction; -import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeCompressionAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeBloomFilterAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeEncodingAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeVersionsAction; -import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.CompactTableAction; -import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction; -import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; -import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction; -import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction; -import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; -import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; -import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction; -import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction; -import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.*; import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy; @@ -64,6 +43,7 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { private long restartRsHoldingMetaSleepTime; private float compactTableRatio; private float compactRandomRegionRatio; + private long decreaseHFileSizeSleepTime; @Override public ChaosMonkey build() { @@ -92,7 +72,8 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { new ChangeEncodingAction(tableName), new ChangeCompressionAction(tableName), new ChangeBloomFilterAction(tableName), - new ChangeVersionsAction(tableName) + new ChangeVersionsAction(tableName), + new ChangeSplitPolicyAction(tableName), }; // Destructive actions to mess things around. @@ -105,7 +86,9 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { new RestartActiveMasterAction(restartActiveMasterSleepTime), new RollingBatchRestartRsAction(rollingBatchRestartRSSleepTime, rollingBatchRestartRSRatio), - new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime) + new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime), + new DecreaseMaxHFileSizeAction(decreaseHFileSizeSleepTime, tableName), + new SplitAllRegionOfTableAction(tableName), }; // Action to log more info for debugging @@ -169,5 +152,8 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { compactRandomRegionRatio = Float.parseFloat(this.properties.getProperty( MonkeyConstants.COMPACT_RANDOM_REGION_RATIO, MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO + "")); + decreaseHFileSizeSleepTime = Long.parseLong(this.properties.getProperty( + MonkeyConstants.DECREASE_HFILE_SIZE_SLEEP_TIME, + MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME + "")); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java index befb2fa..03471ab 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,22 +17,7 @@ */ package org.apache.hadoop.hbase.chaos.factories; -import org.apache.hadoop.hbase.chaos.actions.Action; -import org.apache.hadoop.hbase.chaos.actions.AddColumnAction; -import org.apache.hadoop.hbase.chaos.actions.BatchRestartRsAction; -import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.CompactTableAction; -import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction; -import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; -import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction; -import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction; -import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction; -import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction; -import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.*; import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy; @@ -45,28 +30,35 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory { // Actions that could slow down region movement. // These could also get regions stuck if there are issues. - Action[] actions1 = new Action[] { + Action[] actions1 = new Action[]{ new CompactTableAction(tableName, 0.5f), new CompactRandomRegionOfTableAction(tableName, 0.6f), new FlushTableAction(tableName), new FlushRandomRegionOfTableAction(tableName) }; - Action[] actions2 = new Action[] { + Action[] actions2 = new Action[]{ new SplitRandomRegionOfTableAction(tableName), new MergeRandomAdjacentRegionsOfTableAction(tableName), new AddColumnAction(tableName), new RemoveColumnAction(tableName, columnFamilies), - new MoveRegionsOfTableAction(800, 1600, tableName), - new MoveRandomRegionOfTableAction(800, tableName), - new RestartRandomRsAction(60000), - new BatchRestartRsAction(5000, 0.5f), - new RollingBatchRestartRsAction(5000, 1.0f), - new RestartRsHoldingMetaAction(35000) + new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME, + 1600, + tableName), + new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME, + tableName), + new RestartRandomRsAction(MonkeyConstants.DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME), + new BatchRestartRsAction(MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME, 0.5f), + new RollingBatchRestartRsAction(MonkeyConstants.DEFAULT_BATCH_RESTART_RS_SLEEP_TIME, 1.0f), + new RestartRsHoldingMetaAction(MonkeyConstants.DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME), + new ChangeSplitPolicyAction(tableName), + new SplitAllRegionOfTableAction(tableName), + new DecreaseMaxHFileSizeAction(MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME, + tableName), }; // Action to log more info for debugging - Action[] actions3 = new Action[] { + Action[] actions3 = new Action[]{ new DumpClusterStatusAction() }; http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index f3d8360..d23da64 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -525,20 +525,31 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { // Always add these families. Just skip writing to them when we do not test per CF flush. htd.addFamily(new HColumnDescriptor(BIG_FAMILY_NAME)); htd.addFamily(new HColumnDescriptor(TINY_FAMILY_NAME)); - int numberOfServers = admin.getClusterStatus().getServers().size(); - if (numberOfServers == 0) { - throw new IllegalStateException("No live regionservers"); - } - int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY, - HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER); - int totalNumberOfRegions = numberOfServers * regionsPerServer; - LOG.info("Number of live regionservers: " + numberOfServers + ", " + - "pre-splitting table into " + totalNumberOfRegions + " regions " + - "(default regions per server: " + regionsPerServer + ")"); - byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions); + // If we want to pre-split compute how many splits. + if (conf.getBoolean(HBaseTestingUtility.PRESPLIT_TEST_TABLE_KEY, + HBaseTestingUtility.PRESPLIT_TEST_TABLE)) { + int numberOfServers = admin.getClusterStatus().getServers().size(); + if (numberOfServers == 0) { + throw new IllegalStateException("No live regionservers"); + } + int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY, + HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER); + int totalNumberOfRegions = numberOfServers * regionsPerServer; + LOG.info("Number of live regionservers: " + numberOfServers + ", " + + "pre-splitting table into " + totalNumberOfRegions + " regions " + + "(default regions per server: " + regionsPerServer + ")"); + - admin.createTable(htd, splits); + byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions); + + admin.createTable(htd, splits); + } else { + // Looks like we're just letting things play out. + // Create a table with on region by default. + // This will make the splitting work hard. + admin.createTable(htd); + } } } catch (MasterNotRunningException e) { LOG.error("Master not running", e); http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 4282a5e..e6326e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; http://git-wip-us.apache.org/repos/asf/hbase/blob/80190095/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index bc730b8..a26a3bc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -155,8 +155,11 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * The default number of regions per regionserver when creating a pre-split * table. */ - public static final int DEFAULT_REGIONS_PER_SERVER = 5; + public static final int DEFAULT_REGIONS_PER_SERVER = 3; + + public static final String PRESPLIT_TEST_TABLE_KEY = "hbase.test.pre-split-table"; + public static final boolean PRESPLIT_TEST_TABLE = true; /** * Set if we were passed a zkCluster. If so, we won't shutdown zk as * part of general shutdown.