Source code

001/*
-002 * Licensed to the Apache Software Foundation (ASF) under one
-003 * or more contributor license agreements.  See the NOTICE file
-004 * distributed with this work for additional information
-005 * regarding copyright ownership.  The ASF licenses this file
-006 * to you under the Apache License, Version 2.0 (the
-007 * "License"); you may not use this file except in compliance
-008 * with the License.  You may obtain a copy of the License at
-009 *
-010 *     http://www.apache.org/licenses/LICENSE-2.0
-011 *
-012 * Unless required by applicable law or agreed to in writing, software
-013 * distributed under the License is distributed on an "AS IS" BASIS,
-014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-015 * See the License for the specific language governing permissions and
-016 * limitations under the License.
-017 */
-018package org.apache.hadoop.hbase.client;
-019
-020import static org.junit.Assert.assertEquals;
-021import static org.junit.Assert.assertFalse;
-022import static org.junit.Assert.assertNotNull;
-023import static org.junit.Assert.assertNull;
-024import static org.junit.Assert.assertTrue;
-025import static org.junit.Assert.fail;
-026
-027import java.io.IOException;
-028import java.lang.reflect.Field;
-029import java.lang.reflect.Modifier;
-030import java.net.SocketTimeoutException;
-031import java.util.ArrayList;
-032import java.util.List;
-033import java.util.Optional;
-034import java.util.Random;
-035import java.util.concurrent.ExecutorService;
-036import java.util.concurrent.SynchronousQueue;
-037import java.util.concurrent.ThreadPoolExecutor;
-038import java.util.concurrent.TimeUnit;
-039import java.util.concurrent.atomic.AtomicBoolean;
-040import java.util.concurrent.atomic.AtomicInteger;
-041import java.util.concurrent.atomic.AtomicLong;
-042import java.util.concurrent.atomic.AtomicReference;
-043
-044import org.apache.hadoop.conf.Configuration;
-045import org.apache.hadoop.hbase.CategoryBasedTimeout;
-046import org.apache.hadoop.hbase.Cell;
-047import org.apache.hadoop.hbase.HBaseTestingUtility;
-048import org.apache.hadoop.hbase.HConstants;
-049import org.apache.hadoop.hbase.HRegionLocation;
-050import org.apache.hadoop.hbase.HTableDescriptor;
-051import org.apache.hadoop.hbase.RegionLocations;
-052import org.apache.hadoop.hbase.ServerName;
-053import org.apache.hadoop.hbase.TableName;
-054import org.apache.hadoop.hbase.Waiter;
-055import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-056import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
-057import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-058import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-059import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
-060import org.apache.hadoop.hbase.exceptions.DeserializationException;
-061import org.apache.hadoop.hbase.exceptions.RegionMovedException;
-062import org.apache.hadoop.hbase.filter.Filter;
-063import org.apache.hadoop.hbase.filter.FilterBase;
-064import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-065import org.apache.hadoop.hbase.ipc.RpcClient;
-066import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
-067import org.apache.hadoop.hbase.master.HMaster;
-068import org.apache.hadoop.hbase.regionserver.HRegion;
-069import org.apache.hadoop.hbase.regionserver.HRegionServer;
-070import org.apache.hadoop.hbase.regionserver.Region;
-071import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
-072import org.apache.hadoop.hbase.wal.WALEdit;
-073import org.apache.hadoop.hbase.testclassification.LargeTests;
-074import org.apache.hadoop.hbase.util.Bytes;
-075import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-076import org.apache.hadoop.hbase.util.JVMClusterUtil;
-077import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
-078import org.apache.hadoop.hbase.util.Threads;
-079import org.junit.AfterClass;
-080import org.junit.Assert;
-081import org.junit.BeforeClass;
-082import org.junit.Ignore;
-083import org.junit.Rule;
-084import org.junit.Test;
-085import org.junit.experimental.categories.Category;
-086import org.junit.rules.TestName;
-087import org.junit.rules.TestRule;
-088import org.slf4j.Logger;
-089import org.slf4j.LoggerFactory;
-090import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-091
-092/**
-093 * This class is for testing HBaseConnectionManager features
-094 */
-095@Category({LargeTests.class})
-096public class TestHCM {
-097  @Rule public final TestRule timeout = CategoryBasedTimeout.builder()
-098      .withTimeout(this.getClass())
-099      .withLookingForStuckThread(true)
-100      .build();
-101  private static final Logger LOG = LoggerFactory.getLogger(TestHCM.class);
-102  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-103  private static final TableName TABLE_NAME =
-104      TableName.valueOf("test");
-105  private static final TableName TABLE_NAME1 =
-106      TableName.valueOf("test1");
-107  private static final TableName TABLE_NAME2 =
-108      TableName.valueOf("test2");
-109  private static final TableName TABLE_NAME3 =
-110      TableName.valueOf("test3");
-111  private static final byte[] FAM_NAM = Bytes.toBytes("f");
-112  private static final byte[] ROW = Bytes.toBytes("bbb");
-113  private static final byte[] ROW_X = Bytes.toBytes("xxx");
-114  private static Random _randy = new Random();
-115  private static final int RPC_RETRY = 5;
-116
-117  @Rule
-118  public TestName name = new TestName();
-119
-120/**
-121* This copro sleeps 20 second. The first call it fails. The second time, it works.
-122*/
-123  public static class SleepAndFailFirstTime implements RegionCoprocessor, RegionObserver {
-124    static final AtomicLong ct = new AtomicLong(0);
-125    static final String SLEEP_TIME_CONF_KEY =
-126        "hbase.coprocessor.SleepAndFailFirstTime.sleepTime";
-127    static final long DEFAULT_SLEEP_TIME = 20000;
-128    static final AtomicLong sleepTime = new AtomicLong(DEFAULT_SLEEP_TIME);
-129
-130    public SleepAndFailFirstTime() {
-131    }
-132
-133    @Override
-134    public Optional<RegionObserver> getRegionObserver() {
-135      return Optional.of(this);
-136    }
-137
-138    @Override
-139    public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
-140      RegionCoprocessorEnvironment env = c.getEnvironment();
-141      Configuration conf = env.getConfiguration();
-142      sleepTime.set(conf.getLong(SLEEP_TIME_CONF_KEY, DEFAULT_SLEEP_TIME));
-143    }
-144
-145    @Override
-146    public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
-147        final Get get, final List<Cell> results) throws IOException {
-148      Threads.sleep(sleepTime.get());
-149      if (ct.incrementAndGet() == 1) {
-150        throw new IOException("first call I fail");
-151      }
-152    }
-153
-154    @Override
-155    public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
-156        final Put put, final WALEdit edit, final Durability durability) throws IOException {
-157      Threads.sleep(sleepTime.get());
-158      if (ct.incrementAndGet() == 1) {
-159        throw new IOException("first call I fail");
-160      }
-161    }
-162
-163    @Override
-164    public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> e,
-165        final Delete delete,
-166        final WALEdit edit, final Durability durability) throws IOException {
-167      Threads.sleep(sleepTime.get());
-168      if (ct.incrementAndGet() == 1) {
-169        throw new IOException("first call I fail");
-170      }
-171    }
-172
-173    @Override
-174    public Result preIncrement(final ObserverContext<RegionCoprocessorEnvironment> e,
-175        final Increment increment) throws IOException {
-176      Threads.sleep(sleepTime.get());
-177      if (ct.incrementAndGet() == 1) {
-178        throw new IOException("first call I fail");
-179      }
-180      return null;
-181    }
-182
-183  }
-184
-185  public static class SleepCoprocessor implements RegionCoprocessor, RegionObserver {
-186    public static final int SLEEP_TIME = 5000;
-187
-188    @Override
-189    public Optional<RegionObserver> getRegionObserver() {
-190      return Optional.of(this);
-191    }
-192
-193    @Override
-194    public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
-195        final Get get, final List<Cell> results) throws IOException {
-196      Threads.sleep(SLEEP_TIME);
-197    }
-198
-199    @Override
-200    public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
-201        final Put put, final WALEdit edit, final Durability durability) throws IOException {
-202      Threads.sleep(SLEEP_TIME);
-203    }
-204
-205    @Override
-206    public Result preIncrement(final ObserverContext<RegionCoprocessorEnvironment> e,
-207                               final Increment increment) throws IOException {
-208      Threads.sleep(SLEEP_TIME);
-209      return null;
-210    }
-211
-212    @Override
-213    public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> e, final Delete delete,
-214        final WALEdit edit, final Durability durability) throws IOException {
-215      Threads.sleep(SLEEP_TIME);
-216    }
-217
-218  }
-219
-220  @BeforeClass
-221  public static void setUpBeforeClass() throws Exception {
-222    TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true);
-223    // Up the handlers; this test needs more than usual.
-224    TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
-225    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RPC_RETRY);
-226    TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 3);
-227    TEST_UTIL.startMiniCluster(2);
-228
-229  }
-230
-231  @AfterClass public static void tearDownAfterClass() throws Exception {
-232    TEST_UTIL.shutdownMiniCluster();
-233  }
-234
-235  public void testClusterConnection() throws IOException {
-236    ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1,
-237        5, TimeUnit.SECONDS,
-238        new SynchronousQueue<>(),
-239        Threads.newDaemonThreadFactory("test-hcm"));
-240
-241    Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
-242    Connection con2 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), otherPool);
-243    // make sure the internally created ExecutorService is the one passed
-244    assertTrue(otherPool == ((ConnectionImplementation) con2).getCurrentBatchPool());
-245
-246    final TableName tableName = TableName.valueOf(name.getMethodName());
-247    TEST_UTIL.createTable(tableName, FAM_NAM).close();
-248    Table table = con1.getTable(tableName, otherPool);
-249
-250    ExecutorService pool = null;
-251
-252    if(table instanceof HTable) {
-253      HTable t = (HTable) table;
-254      // make sure passing a pool to the getTable does not trigger creation of an internal pool
-255      assertNull("Internal Thread pool should be null",
-256        ((ConnectionImplementation) con1).getCurrentBatchPool());
-257      // table should use the pool passed
-258      assertTrue(otherPool == t.getPool());
-259      t.close();
-260
-261      t = (HTable) con2.getTable(tableName);
-262      // table should use the connectin's internal pool
-263      assertTrue(otherPool == t.getPool());
-264      t.close();
-265
-266      t = (HTable) con2.getTable(tableName);
-267      // try other API too
-268      assertTrue(otherPool == t.getPool());
-269      t.close();
-270
-271      t = (HTable) con2.getTable(tableName);
-272      // try other API too
-273      assertTrue(otherPool == t.getPool());
-274      t.close();
-275
-276      t = (HTable) con1.getTable(tableName);
-277      pool = ((ConnectionImplementation) con1).getCurrentBatchPool();
-278      // make sure an internal pool was created
-279      assertNotNull("An internal Thread pool should have been created", pool);
-280      // and that the table is using it
-281      assertTrue(t.getPool() == pool);
-282      t.close();
-283
-284      t = (HTable) con1.getTable(tableName);
-285      // still using the *same* internal pool
-286      assertTrue(t.getPool() == pool);
-287      t.close();
-288    } else {
-289      table.close();
-290    }
-291
-292    con1.close();
-293
-294    // if the pool was created on demand it should be closed upon connection close
-295    if(pool != null) {
-296      assertTrue(pool.isShutdown());
-297    }
-298
-299    con2.close();
-300    // if the pool is passed, it is not closed
-301    assertFalse(otherPool.isShutdown());
-302    otherPool.shutdownNow();
-303  }
-304
-305  /**
-306   * Naive test to check that Connection#getAdmin returns a properly constructed HBaseAdmin object
-307   * @throws IOException Unable to construct admin
-308   */
-309  @Test
-310  public void testAdminFactory() throws IOException {
-311    Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
-312    Admin admin = con1.getAdmin();
-313    assertTrue(admin.getConnection() == con1);
-314    assertTrue(admin.getConfiguration() == TEST_UTIL.getConfiguration());
-315    con1.close();
-316  }
-317
-318  // Fails too often!  Needs work.  HBASE-12558
-319  // May only fail on non-linux machines? E.g. macosx.
-320  @Ignore @Test (expected = RegionServerStoppedException.class)
-321  // Depends on mulitcast messaging facility that seems broken in hbase2
-322  // See  HBASE-19261 "ClusterStatusPublisher where Master could optionally broadcast notice of
-323  // dead servers is broke"
-324  public void testClusterStatus() throws Exception {
-325    final TableName tableName = TableName.valueOf(name.getMethodName());
-326    byte[] cf = "cf".getBytes();
-327    byte[] rk = "rk1".getBytes();
-328
-329    JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
-330    rs.waitForServerOnline();
-331    final ServerName sn = rs.getRegionServer().getServerName();
-332
-333    Table t = TEST_UTIL.createTable(tableName, cf);
-334    TEST_UTIL.waitTableAvailable(tableName);
-335    TEST_UTIL.waitUntilNoRegionsInTransition();
-336
-337    final ConnectionImplementation hci =  (ConnectionImplementation)TEST_UTIL.getConnection();
-338    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
-339      while (l.getRegionLocation(rk).getPort() != sn.getPort()) {
-340        TEST_UTIL.getAdmin().move(l.getRegionLocation(rk).getRegionInfo().
-341            getEncodedNameAsBytes(), Bytes.toBytes(sn.toString()));
-342        TEST_UTIL.waitUntilNoRegionsInTransition();
-343        hci.clearRegionCache(tableName);
-344      }
-345      Assert.assertNotNull(hci.clusterStatusListener);
-346      TEST_UTIL.assertRegionOnServer(l.getRegionLocation(rk).getRegionInfo(), sn, 20000);
-347    }
-348
-349    Put p1 = new Put(rk);
-350    p1.addColumn(cf, "qual".getBytes(), "val".getBytes());
-351    t.put(p1);
-352
-353    rs.getRegionServer().abort("I'm dead");
-354
-355    // We want the status to be updated. That's a least 10 second
-356    TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
-357      @Override
-358      public boolean evaluate() throws Exception {
-359        return TEST_UTIL.getHBaseCluster().getMaster().getServerManager().
-360            getDeadServers().isDeadServer(sn);
-361      }
-362    });
-363
-364    TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
-365      @Override
-366      public boolean evaluate() throws Exception {
-367        return hci.clusterStatusListener.isDeadServer(sn);
-368      }
-369    });
-370
-371    t.close();
-372    hci.getClient(sn);  // will throw an exception: RegionServerStoppedException
-373  }
-374
-375  /**
-376   * Test that we can handle connection close: it will trigger a retry, but the calls will
-377   *  finish.
-378   */
-379  @Test
-380  public void testConnectionCloseAllowsInterrupt() throws Exception {
-381    testConnectionClose(true);
-382  }
-383
-384  @Test
-385  public void testConnectionNotAllowsInterrupt() throws Exception {
-386    testConnectionClose(false);
-387  }
-388
-389  /**
-390   * Test that an operation can fail if we read the global operation timeout, even if the
-391   * individual timeout is fine. We do that with:
-392   * - client side: an operation timeout of 30 seconds
-393   * - server side: we sleep 20 second at each attempt. The first work fails, the second one
-394   * succeeds. But the client won't wait that much, because 20 + 20 > 30, so the client
-395   * timeouted when the server answers.
-396   */
-397  @Test
-398  public void testGetOperationTimeout() throws Exception {
-399    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-400    hdt.addCoprocessor(SleepAndFailFirstTime.class.getName());
-401    Table table = TEST_UTIL.createTable(hdt, new byte[][]{FAM_NAM}, TEST_UTIL.getConfiguration());
-402    table.setRpcTimeout(Integer.MAX_VALUE);
-403    SleepAndFailFirstTime.ct.set(0);
-404    // Check that it works if the timeout is big enough
-405    table.setOperationTimeout(120 * 1000);
-406    table.get(new Get(FAM_NAM));
-407
-408    // Resetting and retrying. Will fail this time, not enough time for the second try
-409    SleepAndFailFirstTime.ct.set(0);
-410    try {
-411      table.setOperationTimeout(30 * 1000);
-412      table.get(new Get(FAM_NAM));
-413      Assert.fail("We expect an exception here");
-414    } catch (SocketTimeoutException e) {
-415      // The client has a CallTimeout class, but it's not shared.We're not very clean today,
-416      //  in the general case you can expect the call to stop, but the exception may vary.
-417      // In this test however, we're sure that it will be a socket timeout.
-418      LOG.info("We received an exception, as expected ", e);
-419    } catch (IOException e) {
-420      Assert.fail("Wrong exception:" + e.getMessage());
-421    } finally {
-422      table.close();
-423    }
-424  }
-425
-426  @Test
-427  public void testPutOperationTimeout() throws Exception {
-428    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-429    hdt.addCoprocessor(SleepAndFailFirstTime.class.getName());
-430    Table table = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM },TEST_UTIL.getConfiguration());
-431    table.setRpcTimeout(Integer.MAX_VALUE);
-432    SleepAndFailFirstTime.ct.set(0);
-433    // Check that it works if the timeout is big enough
-434    table.setOperationTimeout(120 * 1000);
-435    table.put(new Put(FAM_NAM).addColumn(FAM_NAM, FAM_NAM, FAM_NAM));
-436
-437    // Resetting and retrying. Will fail this time, not enough time for the second try
-438    SleepAndFailFirstTime.ct.set(0);
-439    try {
-440      table.setOperationTimeout(30 * 1000);
-441      table.put(new Put(FAM_NAM).addColumn(FAM_NAM, FAM_NAM, FAM_NAM));
-442      Assert.fail("We expect an exception here");
-443    } catch (SocketTimeoutException e) {
-444      // The client has a CallTimeout class, but it's not shared.We're not very clean today,
-445      //  in the general case you can expect the call to stop, but the exception may vary.
-446      // In this test however, we're sure that it will be a socket timeout.
-447      LOG.info("We received an exception, as expected ", e);
-448    } catch (IOException e) {
-449      Assert.fail("Wrong exception:" + e.getMessage());
-450    } finally {
-451      table.close();
-452    }
-453  }
-454
-455  @Test
-456  public void testDeleteOperationTimeout() throws Exception {
-457    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-458    hdt.addCoprocessor(SleepAndFailFirstTime.class.getName());
-459    Table table = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM },TEST_UTIL.getConfiguration());
-460    table.setRpcTimeout(Integer.MAX_VALUE);
-461    SleepAndFailFirstTime.ct.set(0);
-462    // Check that it works if the timeout is big enough
-463    table.setOperationTimeout(120 * 1000);
-464    table.delete(new Delete(FAM_NAM));
-465
-466    // Resetting and retrying. Will fail this time, not enough time for the second try
-467    SleepAndFailFirstTime.ct.set(0);
-468    try {
-469      table.setOperationTimeout(30 * 1000);
-470      table.delete(new Delete(FAM_NAM));
-471      Assert.fail("We expect an exception here");
-472    } catch (RetriesExhaustedWithDetailsException e) {
-473      // The client has a CallTimeout class, but it's not shared.We're not very clean today,
-474      //  in the general case you can expect the call to stop, but the exception may vary.
-475      // In this test however, we're sure that it will be a socket timeout.
-476      LOG.info("We received an exception, as expected ", e);
-477    } catch (IOException e) {
-478      Assert.fail("Wrong exception:" + e.getMessage());
-479    } finally {
-480      table.close();
-481    }
-482  }
-483
-484  @Test
-485  public void testRpcTimeout() throws Exception {
-486    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-487    hdt.addCoprocessor(SleepCoprocessor.class.getName());
-488    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-489
-490    try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
-491      t.setRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
-492      t.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
-493      t.get(new Get(FAM_NAM));
-494      fail("Get should not have succeeded");
-495    } catch (RetriesExhaustedException e) {
-496      // expected
-497    }
-498
-499    // Again, with configuration based override
-500    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, SleepCoprocessor.SLEEP_TIME / 2);
-501    try (Connection conn = ConnectionFactory.createConnection(c)) {
-502      try (Table t = conn.getTable(hdt.getTableName())) {
-503        t.get(new Get(FAM_NAM));
-504        fail("Get should not have succeeded");
-505      } catch (RetriesExhaustedException e) {
-506        // expected
-507      }
-508    }
-509  }
-510
-511  @Test
-512  public void testIncrementRpcTimeout() throws Exception {
-513    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-514    hdt.addCoprocessor(SleepCoprocessor.class.getName());
-515    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-516
-517    try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
-518      t.setWriteRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
-519      t.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
-520      Increment i = new Increment(FAM_NAM);
-521      i.addColumn(FAM_NAM, FAM_NAM, 1);
-522      t.increment(i);
-523      fail("Write should not have succeeded");
-524    } catch (RetriesExhaustedException e) {
-525      // expected
-526    }
-527
-528    // Again, with configuration based override
-529    c.setInt(HConstants.HBASE_RPC_WRITE_TIMEOUT_KEY, SleepCoprocessor.SLEEP_TIME / 2);
-530    try (Connection conn = ConnectionFactory.createConnection(c)) {
-531      try (Table t = conn.getTable(hdt.getTableName())) {
-532        Increment i = new Increment(FAM_NAM);
-533        i.addColumn(FAM_NAM, FAM_NAM, 1);
-534        t.increment(i);
-535        fail("Write should not have succeeded");
-536      } catch (RetriesExhaustedException e) {
-537        // expected
-538      }
-539    }
-540  }
-541
-542  @Test
-543  public void testDeleteRpcTimeout() throws Exception {
-544    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-545    hdt.addCoprocessor(SleepCoprocessor.class.getName());
-546    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-547
-548    try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
-549      t.setWriteRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
-550      t.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
-551      Delete d = new Delete(FAM_NAM);
-552      d.addColumn(FAM_NAM, FAM_NAM, 1);
-553      t.delete(d);
-554      fail("Write should not have succeeded");
-555    } catch (RetriesExhaustedException e) {
-556      // expected
-557    }
-558
-559  }
-560
-561  @Test
-562  public void testPutRpcTimeout() throws Exception {
-563    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-564    hdt.addCoprocessor(SleepCoprocessor.class.getName());
-565    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-566
-567    try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
-568      t.setWriteRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
-569      t.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
-570      Put p = new Put(FAM_NAM);
-571      p.addColumn(FAM_NAM, FAM_NAM, FAM_NAM);
-572      t.put(p);
-573      fail("Write should not have succeeded");
-574    } catch (RetriesExhaustedException e) {
-575      // expected
-576    }
-577
-578  }
-579
-580  @Test
-581  public void testGetRpcTimeout() throws Exception {
-582    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-583    hdt.addCoprocessor(SleepCoprocessor.class.getName());
-584    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-585
-586    try (Table t = TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }, c)) {
-587      t.setReadRpcTimeout(SleepCoprocessor.SLEEP_TIME / 2);
-588      t.setOperationTimeout(SleepCoprocessor.SLEEP_TIME * 100);
-589      t.get(new Get(FAM_NAM));
-590      fail("Get should not have succeeded");
-591    } catch (RetriesExhaustedException e) {
-592      // expected
-593    }
-594
-595    // Again, with configuration based override
-596    c.setInt(HConstants.HBASE_RPC_READ_TIMEOUT_KEY, SleepCoprocessor.SLEEP_TIME / 2);
-597    try (Connection conn = ConnectionFactory.createConnection(c)) {
-598      try (Table t = conn.getTable(hdt.getTableName())) {
-599        t.get(new Get(FAM_NAM));
-600        fail("Get should not have succeeded");
-601      } catch (RetriesExhaustedException e) {
-602        // expected
-603      }
-604    }
-605  }
-606
-607  /**
-608   * Test starting from 0 index when RpcRetryingCaller calculate the backoff time.
-609   */
-610  @Test
-611  public void testRpcRetryingCallerSleep() throws Exception {
-612    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
-613    hdt.addCoprocessorWithSpec("|" + SleepAndFailFirstTime.class.getName() + "||"
-614        + SleepAndFailFirstTime.SLEEP_TIME_CONF_KEY + "=2000");
-615    TEST_UTIL.createTable(hdt, new byte[][] { FAM_NAM }).close();
-616
-617    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
-618    SleepAndFailFirstTime.ct.set(0);
-619    c.setInt(HConstants.HBASE_CLIENT_PAUSE, 3000);
-620    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 4000);
-621
-622    Connection connection = ConnectionFactory.createConnection(c);
-623    Table table = connection.getTable(TableName.valueOf(name.getMethodName()));
-624    table.setOperationTimeout(8000);
-625    // Check that it works. Because 2s + 3s * RETRY_BACKOFF[0] + 2s < 8s
-626    table.get(new Get(FAM_NAM));
-627
-628    // Resetting and retrying.
-629    SleepAndFailFirstTime.ct.set(0);
-630    try {
-631      table.setOperationTimeout(6000);
-632      // Will fail this time. After sleep, there are not enough time for second retry
-633      // Beacuse 2s + 3s + 2s > 6s
-634      table.get(new Get(FAM_NAM));
-635      Assert.fail("We expect an exception here");
-636    } catch (SocketTimeoutException e) {
-637      LOG.info("We received an exception, as expected ", e);
-638    } catch (IOException e) {
-639      Assert.fail("Wrong exception:" + e.getMessage());
-640    } finally {
-641      table.close();
-642      connection.close();
-643    }
-644  }
-645
-646  @Test
-647  public void testCallableSleep() throws Exception {
-648    long pauseTime;
-649    long baseTime = 100;
-650    final TableName tableName = TableName.valueOf(name.getMethodName());
-651    TEST_UTIL.createTable(tableName, FAM_NAM);
-652    ClientServiceCallable<Object> regionServerCallable = new ClientServiceCallable<Object>(
-653        TEST_UTIL.getConnection(), tableName, ROW,
-654        new RpcControllerFactory(TEST_UTIL.getConfiguration()).newController(), HConstants.PRIORITY_UNSET) {
-655      @Override
-656      protected Object rpcCall() throws Exception {
-657        return null;
-658      }
-659    };
-660
-661    regionServerCallable.prepare(false);
-662    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
-663      pauseTime = regionServerCallable.sleep(baseTime, i);
-664      assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
-665      assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
-666    }
-667
-668    RegionAdminServiceCallable<Object> regionAdminServiceCallable =
-669        new RegionAdminServiceCallable<Object>(
-670        (ClusterConnection) TEST_UTIL.getConnection(),
-671          new RpcControllerFactory(TEST_UTIL.getConfiguration()), tableName, ROW) {
-672      @Override
-673      public Object call(HBaseRpcController controller) throws Exception {
-674        return null;
-675      }
-676    };
-677
-678    regionAdminServiceCallable.prepare(false);
-679    for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
-680      pauseTime = regionAdminServiceCallable.sleep(baseTime, i);
-681      assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
-682      assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
-683    }
-684
-685    MasterCallable<Object> masterCallable = new MasterCallable<Object>(TEST_UTIL.getConnection(),
-686        new RpcControllerFactory(TEST_UTIL.getConfiguration())) {
-687      @Override
-688      protected Object rpcCall() throws Exception {
-689        return null;
-690      }
-691    };
-692    try {
-693      for (int i = 0; i < HConstants.RETRY_BACKOFF.length; i++) {
-694        pauseTime = masterCallable.sleep(baseTime, i);
-695        assertTrue(pauseTime >= (baseTime * HConstants.RETRY_BACKOFF[i]));
-696        assertTrue(pauseTime <= (baseTime * HConstants.RETRY_BACKOFF[i] * 1.01f));
-697      }
-698    } finally {
-699      masterCallable.close();
-700    }
-701  }
-702
-703  private void testConnectionClose(boolean allowsInterrupt) throws Exception {
-704    TableName tableName = TableName.valueOf("HCM-testConnectionClose" + allowsInterrupt);
-705    TEST_UTIL.createTable(tableName, FAM_NAM).close();
-706
-707    boolean previousBalance = TEST_UTIL.getAdmin().setBalancerRunning(false, true);
-708
-709    Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
-710    // We want to work on a separate connection.
-711    c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
-712    c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 100); // retry a lot
-713    c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 1); // don't wait between retries.
-714    c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0); // Server do not really expire
-715    c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
-716    // to avoid the client to be stuck when do the Get
-717    c2.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 10000);
-718    c2.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 10000);
-719    c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000);
-720
-721    Connection connection = ConnectionFactory.createConnection(c2);
-722    final Table table = connection.getTable(tableName);
-723
-724    Put put = new Put(ROW);
-725    put.addColumn(FAM_NAM, ROW, ROW);
-726    table.put(put);
-727
-728    // 4 steps: ready=0; doGets=1; mustStop=2; stopped=3
-729    final AtomicInteger step = new AtomicInteger(0);
-730
-731    final AtomicReference<Throwable> failed = new AtomicReference<>(null);
-732    Thread t = new Thread("testConnectionCloseThread") {
-733      @Override
-734      public void run() {
-735        int done = 0;
-736        try {
-737          step.set(1);
-738          while (step.get() == 1) {
-739            Get get = new Get(ROW);
-740            table.get(get);
-741            done++;
-742            if (done % 100 == 0)
-743              LOG.info("done=" + done);
-744            // without the sleep, will cause the exception for too many files in
-745            // org.apache.hadoop.hdfs.server.datanode.DataXceiver
-746            Thread.sleep(100);
-747          }
-748        } catch (Throwable t) {
-749          failed.set(t);
-750          LOG.error(t.toString(), t);
-751        }
-752        step.set(3);
-753      }
-754    };
-755    t.start();
-756    TEST_UTIL.waitFor(20000, new Waiter.Predicate<Exception>() {
-757      @Override
-758      public boolean evaluate() throws Exception {
-759        return step.get() == 1;
-760      }
-761    });
-762
-763    ServerName sn;
-764    try(RegionLocator rl = connection.getRegionLocator(tableName)) {
-765      sn = rl.getRegionLocation(ROW).getServerName();
-766    }
-767    ConnectionImplementation conn =
-768        (ConnectionImplementation) connection;
-769    RpcClient rpcClient = conn.getRpcClient();
-770
-771    LOG.info("Going to cancel connections. connection=" + conn.toString() + ", sn=" + sn);
-772    for (int i = 0; i < 5000; i++) {
-773      rpcClient.cancelConnections(sn);
-774      Thread.sleep(5);
-775    }
-776
-777    step.compareAndSet(1, 2);
-778    // The test may fail here if the thread doing the gets is stuck. The way to find
-779    //  out what's happening is to look for the thread named 'testConnectionCloseThread'
-780    TEST_UTIL.waitFor(40000, new Waiter.Predicate<Exception>() {
-781      @Override
-782      public boolean evaluate() throws Exception {
-783        return step.get() == 3;
-784      }
-785    });
-786    table.close();
-787    connection.close();
-788    Assert.assertTrue("Unexpected exception is " + failed.get(), failed.get() == null);
-789    TEST_UTIL.getAdmin().setBalancerRunning(previousBalance, true);
-790  }
-791
-792  /**
-793   * Test that connection can become idle without breaking everything.
-794   */
-795  @Test
-796  public void testConnectionIdle() throws Exception {
-797    final TableName tableName = TableName.valueOf(name.getMethodName());
-798    TEST_UTIL.createTable(tableName, FAM_NAM).close();
-799    int idleTime =  20000;
-800    boolean previousBalance = TEST_UTIL.getAdmin().setBalancerRunning(false, true);
-801
-802    Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
-803    // We want to work on a separate connection.
-804    c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
-805    c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // Don't retry: retry = test failed
-806    c2.setInt(RpcClient.IDLE_TIME, idleTime);
-807
-808    Connection connection = ConnectionFactory.createConnection(c2);
-809    final Table table = connection.getTable(tableName);
-810
-811    Put put = new Put(ROW);
-812    put.addColumn(FAM_NAM, ROW, ROW);
-813    table.put(put);
-814
-815    ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
-816    mee.setValue(System.currentTimeMillis());
-817    EnvironmentEdgeManager.injectEdge(mee);
-818    LOG.info("first get");
-819    table.get(new Get(ROW));
-820
-821    LOG.info("first get - changing the time & sleeping");
-822    mee.incValue(idleTime + 1000);
-823    Thread.sleep(1500); // we need to wait a little for the connection to be seen as idle.
-824                        // 1500 = sleep time in RpcClient#waitForWork + a margin
-825
-826    LOG.info("second get - connection has been marked idle in the middle");
-827    // To check that the connection actually became idle would need to read some private
-828    //  fields of RpcClient.
-829    table.get(new Get(ROW));
-830    mee.incValue(idleTime + 1000);
-831
-832    LOG.info("third get - connection is idle, but the reader doesn't know yet");
-833    // We're testing here a special case:
-834    //  time limit reached BUT connection not yet reclaimed AND a new call.
-835    //  in this situation, we don't close the connection, instead we use it immediately.
-836    // If we're very unlucky we can have a race condition in the test: the connection is already
-837    //  under closing when we do the get, so we have an exception, and we don't retry as the
-838    //  retry number is 1. The probability is very very low, and seems acceptable for now. It's
-839    //  a test issue only.
-840    table.get(new Get(ROW));
-841
-842    LOG.info("we're done - time will change back");
-843
-844    table.close();
-845
-846    connection.close();
-847    EnvironmentEdgeManager.reset();
-848    TEST_UTIL.getAdmin().setBalancerRunning(previousBalance, true);
-849  }
-850
-851    /**
-852     * Test that the connection to the dead server is cut immediately when we receive the
-853     *  notification.
-854     * @throws Exception
-855     */
-856  @Test
-857  public void testConnectionCut() throws Exception {
-858    final TableName tableName = TableName.valueOf(name.getMethodName());
-859
-860    TEST_UTIL.createTable(tableName, FAM_NAM).close();
-861    boolean previousBalance = TEST_UTIL.getAdmin().setBalancerRunning(false, true);
-862
-863    Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
-864    // We want to work on a separate connection.
-865    c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
-866    // try only once w/o any retry
-867    c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);
-868    c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 30 * 1000);
-869
-870    final Connection connection = ConnectionFactory.createConnection(c2);
-871    final Table table = connection.getTable(tableName);
-872
-873    Put p = new Put(FAM_NAM);
-874    p.addColumn(FAM_NAM, FAM_NAM, FAM_NAM);
-875    table.put(p);
-876
-877    final ConnectionImplementation hci =  (ConnectionImplementation) connection;
-878
-879    final HRegionLocation loc;
-880    try(RegionLocator rl = connection.getRegionLocator(tableName)) {
-88