hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hbase git commit: HBASE-13470 High level Integration test for master DDL operations (Sophia Feng)
Date Wed, 17 Jun 2015 05:26:02 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 1eb8ac6fe -> 8b9eb5f1c


HBASE-13470 High level Integration test for master DDL operations (Sophia Feng)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b9eb5f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b9eb5f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b9eb5f1

Branch: refs/heads/branch-1.2
Commit: 8b9eb5f1c52e8eb5c8b2937f1f50ab6869f8bf7f
Parents: 1eb8ac6
Author: Enis Soztutar <enis@apache.org>
Authored: Tue Jun 16 22:22:35 2015 -0700
Committer: Enis Soztutar <enis@apache.org>
Committed: Tue Jun 16 22:24:46 2015 -0700

----------------------------------------------------------------------
 .../hbase/IntegrationTestDDLMasterFailover.java | 827 +++++++++++++++++++
 .../factories/MasterKillingMonkeyFactory.java   |  71 ++
 .../hbase/chaos/factories/MonkeyFactory.java    |   2 +
 3 files changed, 900 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8b9eb5f1/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java
new file mode 100644
index 0000000..f460aa9
--- /dev/null
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.java
@@ -0,0 +1,827 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HBaseFsck;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ *
+ * Integration test that verifies Procedure V2. <br/><br/>
+ *
+ * DDL operations should go through (rollforward or rollback) when primary master is killed
by
+ * ChaosMonkey (default MASTER_KILLING)<br/><br/>
+ *
+ * Multiple Worker threads are started to randomly do the following Actions in loops:<br/>
+ * Actions generating and populating tables:
+ * <ul>
+ *     <li>CreateTableAction</li>
+ *     <li>DisableTableAction</li>
+ *     <li>EnableTableAction</li>
+ *     <li>DeleteTableAction</li>
+ *     <li>AddRowAction</li>
+ * </ul>
+ * Actions performing DDL operations:
+ * <ul>
+ *     <li>AddColumnFamilyAction</li>
+ *     <li>AlterColumnFamilyVersionsAction</li>
+ *     <li>AlterColumnFamilyEncodingAction</li>
+ *     <li>DeleteColumnFamilyAction</li>
+ * </ul>
+ * <br/>
+ *
+ * The threads run for a period of time (default 20 minutes) then are stopped at the end
of
+ * runtime. Verification is performed towards those checkpoints:
+ * <ol>
+ *     <li>No Actions throw Exceptions.</li>
+ *     <li>No inconsistencies are detected in hbck.</li>
+ * </ol>
+ *
+ * <p>
+ * This test should be run by the hbase user since it invokes hbck at the end
+ * </p><p>
+ * Usage:
+ *  hbase org.apache.hadoop.hbase.IntegrationTestDDLMasterFailover
+ *    -Dhbase.IntegrationTestDDLMasterFailover.runtime=1200000
+ *    -Dhbase.IntegrationTestDDLMasterFailover.numThreads=20
+ *    -Dhbase.IntegrationTestDDLMasterFailover.numRegions=50 --monkey masterKilling
+ */
+
+@Category(IntegrationTests.class)
+public class IntegrationTestDDLMasterFailover extends IntegrationTestBase {
+
+  private static final Log LOG = LogFactory.getLog(IntegrationTestDDLMasterFailover.class);
+
+  private static final int SERVER_COUNT = 1; // number of slaves for the smallest cluster
+
+  protected static final long DEFAULT_RUN_TIME = 20 * 60 * 1000;
+
+  protected static final int DEFAULT_NUM_THREADS = 20;
+
+  protected static final int DEFAULT_NUM_REGIONS = 50; // number of regions in pre-split
tables
+
+  protected HBaseCluster cluster;
+
+  protected Connection connection;
+
+  /**
+   * A soft limit on how long we should run
+   */
+  protected static final String RUN_TIME_KEY = "hbase.%s.runtime";
+  protected static final String NUM_THREADS_KEY = "hbase.%s.numThreads";
+  protected static final String NUM_REGIONS_KEY = "hbase.%s.numRegions";
+
+  protected AtomicBoolean running = new AtomicBoolean(true);
+
+  protected AtomicBoolean create_table = new AtomicBoolean(true);
+
+  protected int numThreads, numRegions;
+
+  ConcurrentHashMap<TableName, HTableDescriptor> enabledTables =
+      new ConcurrentHashMap<TableName, HTableDescriptor>();
+
+  ConcurrentHashMap<TableName, HTableDescriptor> disabledTables =
+      new ConcurrentHashMap<TableName, HTableDescriptor>();
+
+  ConcurrentHashMap<TableName, HTableDescriptor> deletedTables =
+      new ConcurrentHashMap<TableName, HTableDescriptor>();
+
+  @Override
+  public void setUpCluster() throws Exception {
+    util = getTestingUtil(getConf());
+    LOG.debug("Initializing/checking cluster has " + SERVER_COUNT + " servers");
+    util.initializeCluster(getMinServerCount());
+    LOG.debug("Done initializing/checking cluster");
+    cluster = util.getHBaseClusterInterface();
+  }
+
+  @Override
+  public void cleanUpCluster() throws Exception {
+    Admin admin = util.getHBaseAdmin();
+    admin.disableTables("ittable-\\d+");
+    admin.deleteTables("ittable-\\d+");
+    Connection connection = getConnection();
+    connection.close();
+    super.cleanUpCluster();
+  }
+
+  protected int getMinServerCount() {
+    return SERVER_COUNT;
+  }
+
+  protected synchronized void setConnection(Connection connection){
+    this.connection = connection;
+  }
+
+  protected synchronized Connection getConnection(){
+    if (this.connection == null) {
+      try {
+        Connection connection = ConnectionFactory.createConnection(getConf());
+        setConnection(connection);
+      } catch (IOException e) {
+        LOG.fatal("Failed to establish connection.", e);
+      }
+    }
+    return connection;
+  }
+
+  protected void verifyTables() throws  IOException{
+    Connection connection = getConnection();
+    Admin admin = connection.getAdmin();
+    // iterating concurrent map
+    for (TableName tableName : enabledTables.keySet()){
+      Assert.assertTrue("Table: " + tableName + " in enabledTables is not enabled",
+          admin.isTableEnabled(tableName));
+    }
+    for (TableName tableName : disabledTables.keySet()){
+      Assert.assertTrue("Table: " + tableName + " in disabledTables is not disabled",
+          admin.isTableDisabled(tableName));
+    }
+    for (TableName tableName : deletedTables.keySet()){
+      Assert.assertFalse("Table: " + tableName + " in deletedTables is not deleted",
+          admin.tableExists(tableName));
+    }
+    admin.close();
+  }
+
+  @Test
+  public void testAsUnitTest() throws Exception {
+    runTest();
+  }
+
+  @Override
+  public int runTestFromCommandLine() throws Exception {
+    int ret = runTest();
+    return ret;
+  }
+
+  private abstract class MasterAction{
+    Connection connection = getConnection();
+
+    abstract void perform() throws IOException;
+  }
+
+  private abstract class TableAction extends  MasterAction{
+    // TableAction has implemented selectTable() shared by multiple table Actions
+    protected HTableDescriptor selectTable(ConcurrentHashMap<TableName, HTableDescriptor>
tableMap)
+    {
+      // randomly select table from tableMap
+      if (tableMap.isEmpty()){
+        return null;
+      }
+      // synchronization to prevent removal from multiple threads
+      synchronized (tableMap){
+        ArrayList<TableName> tableList = new ArrayList<TableName>(tableMap.keySet());
+        TableName randomKey = tableList.get(RandomUtils.nextInt(tableList.size()));
+        HTableDescriptor randomHtd = tableMap.get(randomKey);
+        // remove from tableMap
+        tableMap.remove(randomKey);
+        return randomHtd;
+      }
+    }
+  }
+
+  private class CreateTableAction extends TableAction {
+
+    @Override
+    void perform() throws IOException {
+      Admin admin = connection.getAdmin();
+      try {
+        HTableDescriptor htd = createTableDesc();
+        TableName tableName = htd.getTableName();
+        if ( admin.tableExists(tableName)){
+          return;
+        }
+        String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName());
+        numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS);
+        byte[] startKey = Bytes.toBytes("row-0000000000");
+        byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE);
+        LOG.info("Creating table:" + htd);
+        admin.createTable(htd, startKey, endKey, numRegions);
+        Assert.assertTrue("Table: " + htd + " was not created", admin.tableExists(tableName));
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        enabledTables.put(tableName, freshTableDesc);
+        LOG.info("Created table:" + freshTableDesc);
+      } catch (Exception e){
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO workaround
+        // when master failover happens during CREATE_TABLE, client will do RPC retry and
get TableExistsException
+        // ignore for now till better resolution
+        if (e instanceof TableExistsException) {
+          LOG.warn("Caught TableExistsException in action: " + this.getClass(), e);
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+
+    private HTableDescriptor createTableDesc() {
+      String tableName = "ittable-" + String.format("%010d",
+        RandomUtils.nextInt(Integer.MAX_VALUE));
+      String familyName = "cf-" + Math.abs(RandomUtils.nextInt());
+      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
+      // add random column family
+      htd.addFamily(new HColumnDescriptor(familyName));
+      return htd;
+    }
+  }
+
+  private class DisableTableAction extends TableAction {
+
+    @Override
+    void perform() throws IOException {
+
+      HTableDescriptor selected = selectTable(enabledTables);
+      if (selected == null) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        TableName tableName = selected.getTableName();
+        LOG.info("Disabling table :" + selected);
+        admin.disableTable(tableName);
+        Assert.assertTrue("Table: " + selected + " was not disabled",
+            admin.isTableDisabled(tableName));
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        disabledTables.put(tableName, freshTableDesc);
+        LOG.info("Disabled table :" + freshTableDesc);
+      } catch (Exception e){
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO workaround
+        // loose restriction for TableNotDisabledException/TableNotEnabledException thrown
in sync
+        // operations
+        // 1) when enable/disable starts, the table state is changed to ENABLING/DISABLING
(ZK node
+        // in 1.x), which will be further changed to ENABLED/DISABLED once the operation
completes
+        // 2) if master failover happens in the middle of the enable/disable operation, the
new
+        // master will try to recover the tables in ENABLING/DISABLING state, as programmed
in
+        // AssignmentManager#recoverTableInEnablingState() and
+        // AssignmentManager#recoverTableInDisablingState()
+        // 3) after the new master initialization completes, the procedure tries to re-do
the
+        // enable/disable operation, which was already done. Ignore those exceptions before
change
+        // of behaviors of AssignmentManager in presence of PV2
+        if (e instanceof TableNotEnabledException) {
+          LOG.warn("Caught TableNotEnabledException in action: " + this.getClass());
+          e.printStackTrace();
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private class EnableTableAction extends TableAction {
+
+    @Override
+    void perform() throws IOException {
+
+      HTableDescriptor selected = selectTable(disabledTables);
+      if (selected == null ) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        TableName tableName = selected.getTableName();
+        LOG.info("Enabling table :" + selected);
+        admin.enableTable(tableName);
+        Assert.assertTrue("Table: " + selected + " was not enabled",
+            admin.isTableEnabled(tableName));
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        enabledTables.put(tableName, freshTableDesc);
+        LOG.info("Enabled table :" + freshTableDesc);
+      } catch (Exception e){
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO workaround
+        // loose restriction for TableNotDisabledException/TableNotEnabledException thrown
in sync
+        // operations 1) when enable/disable starts, the table state is changed to
+        // ENABLING/DISABLING (ZK node in 1.x), which will be further changed to ENABLED/DISABLED
+        // once the operation completes 2) if master failover happens in the middle of the
+        // enable/disable operation, the new master will try to recover the tables in
+        // ENABLING/DISABLING state, as programmed in
+        // AssignmentManager#recoverTableInEnablingState() and
+        // AssignmentManager#recoverTableInDisablingState()
+        // 3) after the new master initialization completes, the procedure tries to re-do
the
+        // enable/disable operation, which was already done. Ignore those exceptions before
+        // change of behaviors of AssignmentManager in presence of PV2
+        if (e instanceof TableNotDisabledException) {
+          LOG.warn("Caught TableNotDisabledException in action: " + this.getClass());
+          e.printStackTrace();
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private class DeleteTableAction extends TableAction {
+
+    @Override
+    void perform() throws IOException {
+
+      HTableDescriptor selected = selectTable(disabledTables);
+      if (selected == null) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        TableName tableName = selected.getTableName();
+        LOG.info("Deleting table :" + selected);
+        admin.deleteTable(tableName);
+        Assert.assertFalse("Table: " + selected + " was not deleted",
+                admin.tableExists(tableName));
+        deletedTables.put(tableName, selected);
+        LOG.info("Deleted table :" + selected);
+      } catch (Exception e){
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO workaround
+        // when master failover happens during DELETE_TABLE, client will do RPC retry and
get
+        // TableNotFoundException ignore for now till better resolution
+        if (e instanceof TableNotFoundException) {
+          LOG.warn("Caught TableNotFoundException in action: " + this.getClass());
+          e.printStackTrace();
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+
+  private abstract class ColumnAction extends TableAction{
+    // ColumnAction has implemented selectFamily() shared by multiple family Actions
+    protected HColumnDescriptor selectFamily(HTableDescriptor htd) {
+      if (htd == null) {
+        return null;
+      }
+      HColumnDescriptor[] families = htd.getColumnFamilies();
+      if (families.length == 0){
+        LOG.info("No column families in table: " + htd);
+        return null;
+      }
+      HColumnDescriptor randomCfd = families[RandomUtils.nextInt(families.length)];
+      return randomCfd;
+    }
+  }
+
+  private class AddColumnFamilyAction extends ColumnAction {
+
+    @Override
+    void perform() throws IOException {
+      HTableDescriptor selected = selectTable(disabledTables);
+      if (selected == null) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        HColumnDescriptor cfd = createFamilyDesc();
+        if (selected.hasFamily(cfd.getName())){
+          LOG.info(new String(cfd.getName()) + " already exists in table "
+              + selected.getTableName());
+          return;
+        }
+        TableName tableName = selected.getTableName();
+        LOG.info("Adding column family: " + cfd + " to table: " + tableName);
+        admin.addColumn(tableName, cfd);
+        // assertion
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        Assert.assertTrue("Column family: " + cfd + " was not added",
+            freshTableDesc.hasFamily(cfd.getName()));
+        LOG.info("Added column family: " + cfd + " to table: " + tableName);
+        disabledTables.put(tableName, freshTableDesc);
+      } catch (Exception e){
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO HBASE-13415
+        // loose restriction for InvalidFamilyOperationException thrown in async operations
before
+        // HBASE-13415 completes when failover happens, multiple procids may be created from
the
+        // same request when 1 procedure succeeds, the others would complain about family
already
+        // exists
+        if (e instanceof InvalidFamilyOperationException) {
+          LOG.warn("Caught InvalidFamilyOperationException in action: " + this.getClass());
+          e.printStackTrace();
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+
+    private HColumnDescriptor createFamilyDesc() {
+      String familyName = "cf-" + String.format("%010d", RandomUtils.nextInt(Integer.MAX_VALUE));
+      HColumnDescriptor cfd = new HColumnDescriptor(familyName);
+      return cfd;
+    }
+  }
+
+  private class AlterFamilyVersionsAction extends ColumnAction {
+
+    @Override
+    void perform() throws IOException {
+      HTableDescriptor selected = selectTable(disabledTables);
+      if (selected == null) {
+        return;
+      }
+      HColumnDescriptor columnDesc = selectFamily(selected);
+      if (columnDesc == null){
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      int versions = RandomUtils.nextInt(10) + 3;
+      try {
+        TableName tableName = selected.getTableName();
+        LOG.info("Altering versions of column family: " + columnDesc + " to: " + versions
+
+            " in table: " + tableName);
+        columnDesc.setMinVersions(versions);
+        columnDesc.setMaxVersions(versions);
+        admin.modifyTable(tableName, selected);
+        // assertion
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName());
+        Assert.assertEquals("Column family: " + columnDesc + " was not altered",
+            freshColumnDesc.getMaxVersions(), versions);
+        Assert.assertEquals("Column family: " + freshColumnDesc + " was not altered",
+            freshColumnDesc.getMinVersions(), versions);
+        LOG.info("Altered versions of column family: " + columnDesc + " to: " + versions
+
+            " in table: " + tableName);
+        disabledTables.put(tableName, freshTableDesc);
+      } catch (Exception e) {
+        LOG.warn("Caught exception in action: " + this.getClass());
+        throw e;
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private class AlterFamilyEncodingAction extends ColumnAction {
+
+    @Override
+    void perform() throws IOException {
+      HTableDescriptor selected = selectTable(disabledTables);
+      if (selected == null) {
+        return;
+      }
+      HColumnDescriptor columnDesc = selectFamily(selected);
+      if (columnDesc == null){
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        TableName tableName = selected.getTableName();
+        // possible DataBlockEncoding ids
+        int[] possibleIds = {0, 2, 3, 4, 6};
+        short id = (short) possibleIds[RandomUtils.nextInt(possibleIds.length)];
+        LOG.info("Altering encoding of column family: " + columnDesc + " to: " + id +
+            " in table: " + tableName);
+        columnDesc.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id));
+        admin.modifyTable(tableName, selected);
+        // assertion
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName());
+        Assert.assertEquals("Encoding of column family: " + columnDesc + " was not altered",
+            freshColumnDesc.getDataBlockEncoding().getId(), id);
+        LOG.info("Altered encoding of column family: " + freshColumnDesc + " to: " + id +
+            " in table: " + tableName);
+        disabledTables.put(tableName, freshTableDesc);
+      } catch (Exception e) {
+        LOG.warn("Caught exception in action: " + this.getClass());
+        throw e;
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private class DeleteColumnFamilyAction extends ColumnAction {
+
+    @Override
+    void perform() throws IOException {
+      HTableDescriptor selected = selectTable(disabledTables);
+      HColumnDescriptor cfd = selectFamily(selected);
+      if (selected == null || cfd == null) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      try {
+        if (selected.getColumnFamilies().length < 2) {
+          LOG.info("No enough column families to delete in table " + selected.getTableName());
+          return;
+        }
+        TableName tableName = selected.getTableName();
+        LOG.info("Deleting column family: " + cfd + " from table: " + tableName);
+        admin.deleteColumn(tableName, cfd.getName());
+        // assertion
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        Assert.assertFalse("Column family: " + cfd + " was not added",
+            freshTableDesc.hasFamily(cfd.getName()));
+        LOG.info("Deleted column family: " + cfd + " from table: " + tableName);
+        disabledTables.put(tableName, freshTableDesc);
+      } catch (Exception e) {
+        LOG.warn("Caught exception in action: " + this.getClass());
+        // TODO HBASE-13415
+        // loose restriction for InvalidFamilyOperationException thrown in async operations
before
+        // HBASE-13415 completes when failover happens, multiple procids may be created from
the
+        //  same request when 1 procedure succeeds, the others would complain about family
not
+        // exists
+        if (e instanceof InvalidFamilyOperationException) {
+          LOG.warn("Caught InvalidFamilyOperationException in action: " + this.getClass());
+          e.printStackTrace();
+        } else {
+          throw e;
+        }
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private class AddRowAction extends ColumnAction {
+    // populate tables
+    @Override
+    void perform() throws IOException {
+      HTableDescriptor selected = selectTable(enabledTables);
+      if (selected == null ) {
+        return;
+      }
+
+      Admin admin = connection.getAdmin();
+      TableName tableName = selected.getTableName();
+      try (Table table = connection.getTable(tableName)){
+        ArrayList<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>(admin.getTableRegions(
+            selected.getTableName()));
+        int numRegions = regionInfos.size();
+        // average number of rows to be added per action to each region
+        int average_rows = 1;
+        int numRows = average_rows * numRegions;
+        LOG.info("Adding " + numRows + " rows to table: " + selected);
+        for (int i = 0; i < numRows; i++){
+          // nextInt(Integer.MAX_VALUE)) to return positive numbers only
+          byte[] rowKey = Bytes.toBytes(
+              "row-" + String.format("%010d", RandomUtils.nextInt(Integer.MAX_VALUE)));
+          HColumnDescriptor cfd = selectFamily(selected);
+          if (cfd == null){
+            return;
+          }
+          byte[] family = cfd.getName();
+          byte[] qualifier = Bytes.toBytes("col-" + RandomUtils.nextInt(Integer.MAX_VALUE)
% 10);
+          byte[] value = Bytes.toBytes("val-" + RandomStringUtils.randomAlphanumeric(10));
+          Put put = new Put(rowKey);
+          put.addColumn(family, qualifier, value);
+          table.put(put);
+        }
+        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
+        enabledTables.put(tableName, freshTableDesc);
+        LOG.info("Added " + numRows + " rows to table: " + selected);
+      } catch (Exception e) {
+        LOG.warn("Caught exception in action: " + this.getClass());
+        throw e;
+      } finally {
+        admin.close();
+      }
+      verifyTables();
+    }
+  }
+
+  private enum ACTION {
+    CREATE_TABLE,
+    DISABLE_TABLE,
+    ENABLE_TABLE,
+    DELETE_TABLE,
+    ADD_COLUMNFAMILY,
+    DELETE_COLUMNFAMILY,
+    ALTER_FAMILYVERSIONS,
+    ALTER_FAMILYENCODING,
+    ADD_ROW
+  }
+
+  private class Worker extends Thread {
+
+    private Exception savedException;
+
+    private ACTION action;
+
+    @Override
+    public void run() {
+      while (running.get()) {
+        // select random action
+        ACTION selectedAction = ACTION.values()[RandomUtils.nextInt() % ACTION.values().length];
+        this.action = selectedAction;
+        LOG.info("Performing Action: " + selectedAction);
+
+        try {
+          switch (selectedAction) {
+          case CREATE_TABLE:
+            // stop creating new tables in the later stage of the test to avoid too many
empty
+            // tables
+            if (create_table.get()) {
+              new CreateTableAction().perform();
+            }
+            break;
+          case ADD_ROW:
+            new AddRowAction().perform();
+            break;
+          case DISABLE_TABLE:
+            new DisableTableAction().perform();
+            break;
+          case ENABLE_TABLE:
+            new EnableTableAction().perform();
+            break;
+          case DELETE_TABLE:
+            // reduce probability of deleting table to 20%
+            if (RandomUtils.nextInt(100) < 20) {
+              new DeleteTableAction().perform();
+            }
+            break;
+          case ADD_COLUMNFAMILY:
+            new AddColumnFamilyAction().perform();
+            break;
+          case DELETE_COLUMNFAMILY:
+            // reduce probability of deleting column family to 20%
+            if (RandomUtils.nextInt(100) < 20) {
+              new DeleteColumnFamilyAction().perform();
+            }
+            break;
+          case ALTER_FAMILYVERSIONS:
+            new AlterFamilyVersionsAction().perform();
+            break;
+          case ALTER_FAMILYENCODING:
+            new AlterFamilyEncodingAction().perform();
+            break;
+          }
+        } catch (Exception ex) {
+          this.savedException = ex;
+          return;
+        }
+      }
+      LOG.info(this.getName() + " stopped");
+    }
+
+    public Exception getSavedException(){
+      return this.savedException;
+    }
+
+    public ACTION getAction(){
+      return this.action;
+    }
+  }
+
+  private void checkException(List<Worker> workers){
+    if(workers == null || workers.isEmpty())
+      return;
+    for (Worker worker : workers){
+      Exception e = worker.getSavedException();
+      if (e != null) {
+        LOG.error("Found exception in thread: " + worker.getName());
+        e.printStackTrace();
+      }
+      Assert.assertNull("Action failed: " + worker.getAction() + " in thread: "
+          + worker.getName(), e);
+    }
+  }
+
+  private int runTest() throws Exception {
+    LOG.info("Starting the test");
+
+    String runtimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName());
+    long runtime = util.getConfiguration().getLong(runtimeKey, DEFAULT_RUN_TIME);
+
+    String numThreadKey = String.format(NUM_THREADS_KEY, this.getClass().getSimpleName());
+    numThreads = util.getConfiguration().getInt(numThreadKey, DEFAULT_NUM_THREADS);
+
+    ArrayList<Worker> workers = new ArrayList<>();
+    for (int i = 0; i < numThreads; i++) {
+      checkException(workers);
+      Worker worker = new Worker();
+      LOG.info("Launching worker thread " + worker.getName());
+      workers.add(worker);
+      worker.start();
+    }
+
+    Threads.sleep(runtime / 2);
+    LOG.info("Stopping creating new tables");
+    create_table.set(false);
+    Threads.sleep(runtime / 2);
+    LOG.info("Runtime is up");
+    running.set(false);
+
+    checkException(workers);
+
+    for (Worker worker : workers) {
+      worker.join();
+    }
+    LOG.info("All Worker threads stopped");
+
+    // verify
+    LOG.info("Verify actions of all threads succeeded");
+    checkException(workers);
+    LOG.info("Verify states of all tables");
+    verifyTables();
+
+    // RUN HBCK
+
+    HBaseFsck hbck = null;
+    try {
+      LOG.info("Running hbck");
+      hbck = HbckTestingUtil.doFsck(util.getConfiguration(), false);
+      HbckTestingUtil.assertNoErrors(hbck);
+      LOG.info("Finished hbck");
+    } finally {
+      if (hbck != null) {
+        hbck.close();
+      }
+    }
+     return 0;
+  }
+
+  @Override
+  public TableName getTablename() {
+    return null;
+  }
+
+  @Override
+  protected Set<String> getColumnFamilies() {
+    return null;
+  }
+
+  public static void main(String[] args) throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    IntegrationTestingUtility.setUseDistributedCluster(conf);
+    IntegrationTestDDLMasterFailover masterFailover = new IntegrationTestDDLMasterFailover();
+    Connection connection = null;
+    int ret = 1;
+    try {
+      // Initialize connection once, then pass to Actions
+      LOG.debug("Setting up connection ...");
+      connection = ConnectionFactory.createConnection(conf);
+      masterFailover.setConnection(connection);
+      ret = ToolRunner.run(conf, masterFailover, args);
+    } catch (IOException e){
+      LOG.fatal("Failed to establish connection. Aborting test ...", e);
+    } finally {
+      connection = masterFailover.getConnection();
+      if (connection != null){
+        connection.close();
+      }
+      System.exit(ret);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b9eb5f1/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MasterKillingMonkeyFactory.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MasterKillingMonkeyFactory.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MasterKillingMonkeyFactory.java
new file mode 100644
index 0000000..52dec3b
--- /dev/null
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MasterKillingMonkeyFactory.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.chaos.factories;
+
+import org.apache.hadoop.hbase.chaos.actions.Action;
+import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
+import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction;
+import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
+import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
+import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
+
+/**
+ * A chaos monkey to kill the active master periodically. Can be run in single master
+ * or multi master setup.
+ */
+public class MasterKillingMonkeyFactory extends MonkeyFactory {
+
+  private long action1Period;
+  private long action2Period;
+
+  private long restartActiveMasterSleepTime;
+
+  @Override
+  public ChaosMonkey build() {
+    loadProperties();
+
+    // Destructive actions to mess things around.
+    Action[] actions1 = new Action[] {
+        new RestartActiveMasterAction(restartActiveMasterSleepTime),
+    };
+
+    // Action to log more info for debugging
+    Action[] actions2 = new Action[] {
+        new DumpClusterStatusAction()
+    };
+
+    return new PolicyBasedChaosMonkey(util,
+        new PeriodicRandomActionPolicy(action1Period, actions1),
+        new PeriodicRandomActionPolicy(action2Period, actions2));
+  }
+
+  private void loadProperties() {
+
+      action1Period = Long.parseLong(this.properties.getProperty(
+        MonkeyConstants.PERIODIC_ACTION1_PERIOD,
+        MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD + ""));
+      action2Period = Long.parseLong(this.properties.getProperty(
+        MonkeyConstants.PERIODIC_ACTION2_PERIOD,
+        MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD + ""));
+      restartActiveMasterSleepTime = Long.parseLong(this.properties.getProperty(
+        MonkeyConstants.RESTART_ACTIVE_MASTER_SLEEP_TIME,
+        MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME + ""));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b9eb5f1/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java
index f4b1c53..2f65251 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java
@@ -69,6 +69,7 @@ public abstract class MonkeyFactory {
   public static final String SERVER_KILLING = "serverKilling";
   public static final String STRESS_AM = "stressAM";
   public static final String NO_KILL = "noKill";
+  public static final String MASTER_KILLING = "masterKilling";
 
   public static Map<String, MonkeyFactory> FACTORIES = ImmutableMap.<String,MonkeyFactory>builder()
     .put(CALM, new CalmMonkeyFactory())
@@ -77,6 +78,7 @@ public abstract class MonkeyFactory {
     .put(SERVER_KILLING, new ServerKillingMonkeyFactory())
     .put(STRESS_AM, new StressAssignmentManagerMonkeyFactory())
     .put(NO_KILL, new NoKillMonkeyFactory())
+    .put(MASTER_KILLING, new MasterKillingMonkeyFactory())
     .build();
 
   public static MonkeyFactory getFactory(String factoryName) {


Mime
View raw message