From commits-return-65419-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Sat Jan 13 16:32:08 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 2168C18077A for ; Sat, 13 Jan 2018 16:32:06 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 11B60160C1C; Sat, 13 Jan 2018 15:32:06 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 54801160C4F for ; Sat, 13 Jan 2018 16:32:03 +0100 (CET) Received: (qmail 90075 invoked by uid 500); 13 Jan 2018 15:31:56 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 89307 invoked by uid 99); 13 Jan 2018 15:31:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Jan 2018 15:31:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C6FBBF32CB; Sat, 13 Jan 2018 15:31:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 13 Jan 2018 15:31:59 -0000 Message-Id: <584a87c675484d869047d62180518d24@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [10/47] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/554d61c6/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestMultiParallel.MyMasterObserver.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestMultiParallel.MyMasterObserver.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestMultiParallel.MyMasterObserver.html index b4bc1f1..9fe9867 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestMultiParallel.MyMasterObserver.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestMultiParallel.MyMasterObserver.html @@ -66,771 +66,773 @@ 058import org.junit.Assert; 059import org.junit.Before; 060import org.junit.BeforeClass; -061import org.junit.Test; -062import org.junit.experimental.categories.Category; -063import org.slf4j.Logger; -064import org.slf4j.LoggerFactory; -065 -066@Category({MediumTests.class, FlakeyTests.class}) -067public class TestMultiParallel { -068 private static final Logger LOG = LoggerFactory.getLogger(TestMultiParallel.class); -069 -070 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); -071 private static final byte[] VALUE = Bytes.toBytes("value"); -072 private static final byte[] QUALIFIER = Bytes.toBytes("qual"); -073 private static final String FAMILY = "family"; -074 private static final TableName TEST_TABLE = TableName.valueOf("multi_test_table"); -075 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY); -076 private static final byte[] ONE_ROW = Bytes.toBytes("xxx"); -077 private static final byte [][] KEYS = makeKeys(); -078 -079 private static final int slaves = 5; // also used for testing HTable pool size -080 private static Connection CONNECTION; -081 -082 @BeforeClass public static void beforeClass() throws Exception { -083 // Uncomment the following lines if more verbosity is needed for -084 // debugging (see HBASE-12285 for details). -085 //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); -086 //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); -087 //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); -088 UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, -089 KeyValueCodec.class.getCanonicalName()); -090 UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true); -091 UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true); -092 UTIL.getConfiguration() -093 .set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyMasterObserver.class.getName()); -094 UTIL.startMiniCluster(slaves); -095 Table t = UTIL.createMultiRegionTable(TEST_TABLE, Bytes.toBytes(FAMILY)); -096 UTIL.waitTableEnabled(TEST_TABLE); -097 t.close(); -098 CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration()); -099 assertTrue(MyMasterObserver.start.get()); -100 } -101 -102 @AfterClass public static void afterClass() throws Exception { -103 CONNECTION.close(); -104 UTIL.shutdownMiniCluster(); -105 } -106 -107 @Before public void before() throws Exception { -108 final int balanceCount = MyMasterObserver.postBalanceCount.get(); -109 LOG.info("before"); -110 if (UTIL.ensureSomeRegionServersAvailable(slaves)) { -111 // Distribute regions -112 UTIL.getMiniHBaseCluster().getMaster().balance(); -113 // Some plans are created. -114 if (MyMasterObserver.postBalanceCount.get() > balanceCount) { -115 // It is necessary to wait the move procedure to start. -116 // Otherwise, the next wait may pass immediately. -117 UTIL.waitFor(3 * 1000, 100, false, () -> -118 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().hasRegionsInTransition() -119 ); -120 } -121 -122 // Wait until completing balance -123 UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); -124 } -125 LOG.info("before done"); -126 } -127 -128 private static byte[][] makeKeys() { -129 byte [][] starterKeys = HBaseTestingUtility.KEYS; -130 // Create a "non-uniform" test set with the following characteristics: -131 // a) Unequal number of keys per region -132 -133 // Don't use integer as a multiple, so that we have a number of keys that is -134 // not a multiple of the number of regions -135 int numKeys = (int) (starterKeys.length * 10.33F); -136 -137 List<byte[]> keys = new ArrayList<>(); -138 for (int i = 0; i < numKeys; i++) { -139 int kIdx = i % starterKeys.length; -140 byte[] k = starterKeys[kIdx]; -141 byte[] cp = new byte[k.length + 1]; -142 System.arraycopy(k, 0, cp, 0, k.length); -143 cp[k.length] = new Integer(i % 256).byteValue(); -144 keys.add(cp); -145 } -146 -147 // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which -148 // should work) -149 // c) keys are not in sorted order (within a region), to ensure that the -150 // sorting code and index mapping doesn't break the functionality -151 for (int i = 0; i < 100; i++) { -152 int kIdx = i % starterKeys.length; -153 byte[] k = starterKeys[kIdx]; -154 byte[] cp = new byte[k.length + 1]; -155 System.arraycopy(k, 0, cp, 0, k.length); -156 cp[k.length] = new Integer(i % 256).byteValue(); -157 keys.add(cp); -158 } -159 return keys.toArray(new byte [][] {new byte [] {}}); -160 } -161 -162 -163 /** -164 * This is for testing the active number of threads that were used while -165 * doing a batch operation. It inserts one row per region via the batch -166 * operation, and then checks the number of active threads. -167 * For HBASE-3553 -168 * @throws IOException -169 * @throws InterruptedException -170 * @throws NoSuchFieldException -171 * @throws SecurityException -172 */ -173 @Test(timeout=300000) -174 public void testActiveThreadsCount() throws Exception { -175 UTIL.getConfiguration().setLong("hbase.htable.threads.coresize", slaves + 1); -176 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration())) { -177 ThreadPoolExecutor executor = HTable.getDefaultExecutor(UTIL.getConfiguration()); -178 try { -179 try (Table t = connection.getTable(TEST_TABLE, executor)) { -180 List<Put> puts = constructPutRequests(); // creates a Put for every region -181 t.batch(puts, null); -182 HashSet<ServerName> regionservers = new HashSet<>(); -183 try (RegionLocator locator = connection.getRegionLocator(TEST_TABLE)) { -184 for (Row r : puts) { -185 HRegionLocation location = locator.getRegionLocation(r.getRow()); -186 regionservers.add(location.getServerName()); -187 } -188 } -189 assertEquals(regionservers.size(), executor.getLargestPoolSize()); -190 } -191 } finally { -192 executor.shutdownNow(); -193 } -194 } -195 } -196 -197 @Test(timeout=300000) -198 public void testBatchWithGet() throws Exception { -199 LOG.info("test=testBatchWithGet"); -200 Table table = UTIL.getConnection().getTable(TEST_TABLE); -201 -202 // load test data -203 List<Put> puts = constructPutRequests(); -204 table.batch(puts, null); -205 -206 // create a list of gets and run it -207 List<Row> gets = new ArrayList<>(); -208 for (byte[] k : KEYS) { -209 Get get = new Get(k); -210 get.addColumn(BYTES_FAMILY, QUALIFIER); -211 gets.add(get); -212 } -213 Result[] multiRes = new Result[gets.size()]; -214 table.batch(gets, multiRes); -215 -216 // Same gets using individual call API -217 List<Result> singleRes = new ArrayList<>(); -218 for (Row get : gets) { -219 singleRes.add(table.get((Get) get)); -220 } -221 // Compare results -222 Assert.assertEquals(singleRes.size(), multiRes.length); -223 for (int i = 0; i < singleRes.size(); i++) { -224 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER)); -225 Cell[] singleKvs = singleRes.get(i).rawCells(); -226 Cell[] multiKvs = multiRes[i].rawCells(); -227 for (int j = 0; j < singleKvs.length; j++) { -228 Assert.assertEquals(singleKvs[j], multiKvs[j]); -229 Assert.assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(singleKvs[j]), -230 CellUtil.cloneValue(multiKvs[j]))); -231 } -232 } -233 table.close(); -234 } -235 -236 @Test -237 public void testBadFam() throws Exception { -238 LOG.info("test=testBadFam"); -239 Table table = UTIL.getConnection().getTable(TEST_TABLE); -240 -241 List<Row> actions = new ArrayList<>(); -242 Put p = new Put(Bytes.toBytes("row1")); -243 p.addColumn(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value")); -244 actions.add(p); -245 p = new Put(Bytes.toBytes("row2")); -246 p.addColumn(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value")); -247 actions.add(p); -248 -249 // row1 and row2 should be in the same region. +061import org.junit.Ignore; +062import org.junit.Test; +063import org.junit.experimental.categories.Category; +064import org.slf4j.Logger; +065import org.slf4j.LoggerFactory; +066 +067@Ignore // Depends on Master being able to host regions. Needs fixing. +068@Category({MediumTests.class, FlakeyTests.class}) +069public class TestMultiParallel { +070 private static final Logger LOG = LoggerFactory.getLogger(TestMultiParallel.class); +071 +072 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); +073 private static final byte[] VALUE = Bytes.toBytes("value"); +074 private static final byte[] QUALIFIER = Bytes.toBytes("qual"); +075 private static final String FAMILY = "family"; +076 private static final TableName TEST_TABLE = TableName.valueOf("multi_test_table"); +077 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY); +078 private static final byte[] ONE_ROW = Bytes.toBytes("xxx"); +079 private static final byte [][] KEYS = makeKeys(); +080 +081 private static final int slaves = 5; // also used for testing HTable pool size +082 private static Connection CONNECTION; +083 +084 @BeforeClass public static void beforeClass() throws Exception { +085 // Uncomment the following lines if more verbosity is needed for +086 // debugging (see HBASE-12285 for details). +087 //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); +088 //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); +089 //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); +090 UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY, +091 KeyValueCodec.class.getCanonicalName()); +092 UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true); +093 UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true); +094 UTIL.getConfiguration() +095 .set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyMasterObserver.class.getName()); +096 UTIL.startMiniCluster(slaves); +097 Table t = UTIL.createMultiRegionTable(TEST_TABLE, Bytes.toBytes(FAMILY)); +098 UTIL.waitTableEnabled(TEST_TABLE); +099 t.close(); +100 CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration()); +101 assertTrue(MyMasterObserver.start.get()); +102 } +103 +104 @AfterClass public static void afterClass() throws Exception { +105 CONNECTION.close(); +106 UTIL.shutdownMiniCluster(); +107 } +108 +109 @Before public void before() throws Exception { +110 final int balanceCount = MyMasterObserver.postBalanceCount.get(); +111 LOG.info("before"); +112 if (UTIL.ensureSomeRegionServersAvailable(slaves)) { +113 // Distribute regions +114 UTIL.getMiniHBaseCluster().getMaster().balance(); +115 // Some plans are created. +116 if (MyMasterObserver.postBalanceCount.get() > balanceCount) { +117 // It is necessary to wait the move procedure to start. +118 // Otherwise, the next wait may pass immediately. +119 UTIL.waitFor(3 * 1000, 100, false, () -> +120 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().hasRegionsInTransition() +121 ); +122 } +123 +124 // Wait until completing balance +125 UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); +126 } +127 LOG.info("before done"); +128 } +129 +130 private static byte[][] makeKeys() { +131 byte [][] starterKeys = HBaseTestingUtility.KEYS; +132 // Create a "non-uniform" test set with the following characteristics: +133 // a) Unequal number of keys per region +134 +135 // Don't use integer as a multiple, so that we have a number of keys that is +136 // not a multiple of the number of regions +137 int numKeys = (int) (starterKeys.length * 10.33F); +138 +139 List<byte[]> keys = new ArrayList<>(); +140 for (int i = 0; i < numKeys; i++) { +141 int kIdx = i % starterKeys.length; +142 byte[] k = starterKeys[kIdx]; +143 byte[] cp = new byte[k.length + 1]; +144 System.arraycopy(k, 0, cp, 0, k.length); +145 cp[k.length] = new Integer(i % 256).byteValue(); +146 keys.add(cp); +147 } +148 +149 // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which +150 // should work) +151 // c) keys are not in sorted order (within a region), to ensure that the +152 // sorting code and index mapping doesn't break the functionality +153 for (int i = 0; i < 100; i++) { +154 int kIdx = i % starterKeys.length; +155 byte[] k = starterKeys[kIdx]; +156 byte[] cp = new byte[k.length + 1]; +157 System.arraycopy(k, 0, cp, 0, k.length); +158 cp[k.length] = new Integer(i % 256).byteValue(); +159 keys.add(cp); +160 } +161 return keys.toArray(new byte [][] {new byte [] {}}); +162 } +163 +164 +165 /** +166 * This is for testing the active number of threads that were used while +167 * doing a batch operation. It inserts one row per region via the batch +168 * operation, and then checks the number of active threads. +169 * For HBASE-3553 +170 * @throws IOException +171 * @throws InterruptedException +172 * @throws NoSuchFieldException +173 * @throws SecurityException +174 */ +175 @Test(timeout=300000) +176 public void testActiveThreadsCount() throws Exception { +177 UTIL.getConfiguration().setLong("hbase.htable.threads.coresize", slaves + 1); +178 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration())) { +179 ThreadPoolExecutor executor = HTable.getDefaultExecutor(UTIL.getConfiguration()); +180 try { +181 try (Table t = connection.getTable(TEST_TABLE, executor)) { +182 List<Put> puts = constructPutRequests(); // creates a Put for every region +183 t.batch(puts, null); +184 HashSet<ServerName> regionservers = new HashSet<>(); +185 try (RegionLocator locator = connection.getRegionLocator(TEST_TABLE)) { +186 for (Row r : puts) { +187 HRegionLocation location = locator.getRegionLocation(r.getRow()); +188 regionservers.add(location.getServerName()); +189 } +190 } +191 assertEquals(regionservers.size(), executor.getLargestPoolSize()); +192 } +193 } finally { +194 executor.shutdownNow(); +195 } +196 } +197 } +198 +199 @Test(timeout=300000) +200 public void testBatchWithGet() throws Exception { +201 LOG.info("test=testBatchWithGet"); +202 Table table = UTIL.getConnection().getTable(TEST_TABLE); +203 +204 // load test data +205 List<Put> puts = constructPutRequests(); +206 table.batch(puts, null); +207 +208 // create a list of gets and run it +209 List<Row> gets = new ArrayList<>(); +210 for (byte[] k : KEYS) { +211 Get get = new Get(k); +212 get.addColumn(BYTES_FAMILY, QUALIFIER); +213 gets.add(get); +214 } +215 Result[] multiRes = new Result[gets.size()]; +216 table.batch(gets, multiRes); +217 +218 // Same gets using individual call API +219 List<Result> singleRes = new ArrayList<>(); +220 for (Row get : gets) { +221 singleRes.add(table.get((Get) get)); +222 } +223 // Compare results +224 Assert.assertEquals(singleRes.size(), multiRes.length); +225 for (int i = 0; i < singleRes.size(); i++) { +226 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER)); +227 Cell[] singleKvs = singleRes.get(i).rawCells(); +228 Cell[] multiKvs = multiRes[i].rawCells(); +229 for (int j = 0; j < singleKvs.length; j++) { +230 Assert.assertEquals(singleKvs[j], multiKvs[j]); +231 Assert.assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(singleKvs[j]), +232 CellUtil.cloneValue(multiKvs[j]))); +233 } +234 } +235 table.close(); +236 } +237 +238 @Test +239 public void testBadFam() throws Exception { +240 LOG.info("test=testBadFam"); +241 Table table = UTIL.getConnection().getTable(TEST_TABLE); +242 +243 List<Row> actions = new ArrayList<>(); +244 Put p = new Put(Bytes.toBytes("row1")); +245 p.addColumn(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value")); +246 actions.add(p); +247 p = new Put(Bytes.toBytes("row2")); +248 p.addColumn(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value")); +249 actions.add(p); 250 -251 Object [] r = new Object[actions.size()]; -252 try { -253 table.batch(actions, r); -254 fail(); -255 } catch (RetriesExhaustedWithDetailsException ex) { -256 LOG.debug(ex.toString(), ex); -257 // good! -258 assertFalse(ex.mayHaveClusterIssues()); -259 } -260 assertEquals(2, r.length); -261 assertTrue(r[0] instanceof Throwable); -262 assertTrue(r[1] instanceof Result); -263 table.close(); -264 } -265 -266 @Test (timeout=300000) -267 public void testFlushCommitsNoAbort() throws Exception { -268 LOG.info("test=testFlushCommitsNoAbort"); -269 doTestFlushCommits(false); -270 } -271 -272 /** -273 * Only run one Multi test with a forced RegionServer abort. Otherwise, the -274 * unit tests will take an unnecessarily long time to run. -275 * -276 * @throws Exception -277 */ -278 @Test (timeout=360000) -279 public void testFlushCommitsWithAbort() throws Exception { -280 LOG.info("test=testFlushCommitsWithAbort"); -281 doTestFlushCommits(true); -282 } -283 -284 /** -285 * Set table auto flush to false and test flushing commits -286 * @param doAbort true if abort one regionserver in the testing -287 * @throws Exception -288 */ -289 private void doTestFlushCommits(boolean doAbort) throws Exception { -290 // Load the data -291 LOG.info("get new table"); -292 Table table = UTIL.getConnection().getTable(TEST_TABLE); -293 -294 LOG.info("constructPutRequests"); -295 List<Put> puts = constructPutRequests(); -296 table.put(puts); -297 LOG.info("puts"); -298 final int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads() -299 .size(); -300 assert liveRScount > 0; -301 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster() -302 .getLiveRegionServerThreads().get(0); -303 if (doAbort) { -304 liveRS.getRegionServer().abort("Aborting for tests", -305 new Exception("doTestFlushCommits")); -306 // If we wait for no regions being online after we abort the server, we -307 // could ensure the master has re-assigned the regions on killed server -308 // after writing successfully. It means the server we aborted is dead -309 // and detected by matser -310 while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) { -311 Thread.sleep(100); -312 } -313 // try putting more keys after the abort. same key/qual... just validating -314 // no exceptions thrown -315 puts = constructPutRequests(); -316 table.put(puts); -317 } -318 -319 LOG.info("validating loaded data"); -320 validateLoadedData(table); -321 -322 // Validate server and region count -323 List<JVMClusterUtil.RegionServerThread> liveRSs = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); -324 int count = 0; -325 for (JVMClusterUtil.RegionServerThread t: liveRSs) { -326 count++; -327 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer()); -328 } -329 LOG.info("Count=" + count); -330 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort, -331 (doAbort ? (liveRScount - 1) : liveRScount), count); -332 if (doAbort) { -333 UTIL.getMiniHBaseCluster().waitOnRegionServer(0); -334 UTIL.waitFor(15 * 1000, new Waiter.Predicate<Exception>() { -335 @Override -336 public boolean evaluate() throws Exception { -337 // Master is also a regionserver, so the count is liveRScount -338 return UTIL.getMiniHBaseCluster().getMaster() -339 .getClusterMetrics().getLiveServerMetrics().size() == liveRScount; -340 } -341 }); -342 UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition()); -343 } -344 -345 table.close(); -346 LOG.info("done"); -347 } -348 -349 @Test (timeout=300000) -350 public void testBatchWithPut() throws Exception { -351 LOG.info("test=testBatchWithPut"); -352 Table table = CONNECTION.getTable(TEST_TABLE); -353 // put multiple rows using a batch -354 List<Put> puts = constructPutRequests(); -355 -356 Object[] results = new Object[puts.size()]; -357 table.batch(puts, results); -358 validateSizeAndEmpty(results, KEYS.length); -359 -360 if (true) { -361 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size(); -362 assert liveRScount > 0; -363 JVMClusterUtil.RegionServerThread liveRS = -364 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0); -365 liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut")); -366 puts = constructPutRequests(); -367 try { -368 results = new Object[puts.size()]; -369 table.batch(puts, results); -370 } catch (RetriesExhaustedWithDetailsException ree) { -371 LOG.info(ree.getExhaustiveDescription()); -372 table.close(); -373 throw ree; -374 } -375 validateSizeAndEmpty(results, KEYS.length); -376 } -377 -378 validateLoadedData(table); -379 table.close(); -380 } -381 -382 @Test(timeout=300000) -383 public void testBatchWithDelete() throws Exception { -384 LOG.info("test=testBatchWithDelete"); -385 Table table = UTIL.getConnection().getTable(TEST_TABLE); -386 -387 // Load some data -388 List<Put> puts = constructPutRequests(); -389 Object[] results = new Object[puts.size()]; -390 table.batch(puts, results); -391 validateSizeAndEmpty(results, KEYS.length); -392 -393 // Deletes -394 List<Row> deletes = new ArrayList<>(); -395 for (int i = 0; i < KEYS.length; i++) { -396 Delete delete = new Delete(KEYS[i]); -397 delete.addFamily(BYTES_FAMILY); -398 deletes.add(delete); -399 } -400 results= new Object[deletes.size()]; -401 table.batch(deletes, results); -402 validateSizeAndEmpty(results, KEYS.length); -403 -404 // Get to make sure ... -405 for (byte[] k : KEYS) { -406 Get get = new Get(k); -407 get.addColumn(BYTES_FAMILY, QUALIFIER); -408 Assert.assertFalse(table.exists(get)); -409 } -410 table.close(); -411 } -412 -413 @Test(timeout=300000) -414 public void testHTableDeleteWithList() throws Exception { -415 LOG.info("test=testHTableDeleteWithList"); -416 Table table = UTIL.getConnection().getTable(TEST_TABLE); -417 -418 // Load some data -419 List<Put> puts = constructPutRequests(); -420 Object[] results = new Object[puts.size()]; -421 table.batch(puts, results); -422 validateSizeAndEmpty(results, KEYS.length); -423 -424 // Deletes -425 ArrayList<Delete> deletes = new ArrayList<>(); -426 for (int i = 0; i < KEYS.length; i++) { -427 Delete delete = new Delete(KEYS[i]); -428 delete.addFamily(BYTES_FAMILY); -429 deletes.add(delete); -430 } -431 table.delete(deletes); -432 Assert.assertTrue(deletes.isEmpty()); -433 -434 // Get to make sure ... -435 for (byte[] k : KEYS) { -436 Get get = new Get(k); -437 get.addColumn(BYTES_FAMILY, QUALIFIER); -438 Assert.assertFalse(table.exists(get)); -439 } -440 table.close(); -441 } -442 -443 @Test(timeout=300000) -444 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception { -445 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut"); -446 Table table = UTIL.getConnection().getTable(TEST_TABLE); -447 -448 List<Row> puts = new ArrayList<>(); -449 for (int i = 0; i < 100; i++) { -450 Put put = new Put(ONE_ROW); -451 byte[] qual = Bytes.toBytes("column" + i); -452 put.addColumn(BYTES_FAMILY, qual, VALUE); -453 puts.add(put); -454 } -455 Object[] results = new Object[puts.size()]; -456 table.batch(puts, results); -457 -458 // validate -459 validateSizeAndEmpty(results, 100); -460 -461 // get the data back and validate that it is correct -462 List<Row> gets = new ArrayList<>(); -463 for (int i = 0; i < 100; i++) { -464 Get get = new Get(ONE_ROW); -465 byte[] qual = Bytes.toBytes("column" + i); -466 get.addColumn(BYTES_FAMILY, qual); -467 gets.add(get); -468 } -469 -470 Object[] multiRes = new Object[gets.size()]; -471 table.batch(gets, multiRes); -472 -473 int idx = 0; -474 for (Object r : multiRes) { -475 byte[] qual = Bytes.toBytes("column" + idx); -476 validateResult(r, qual, VALUE); -477 idx++; -478 } -479 table.close(); -480 } -481 -482 @Test(timeout=300000) -483 public void testBatchWithIncrementAndAppend() throws Exception { -484 LOG.info("test=testBatchWithIncrementAndAppend"); -485 final byte[] QUAL1 = Bytes.toBytes("qual1"); -486 final byte[] QUAL2 = Bytes.toBytes("qual2"); -487 final byte[] QUAL3 = Bytes.toBytes("qual3"); -488 final byte[] QUAL4 = Bytes.toBytes("qual4"); -489 Table table = UTIL.getConnection().getTable(TEST_TABLE); -490 Delete d = new Delete(ONE_ROW); -491 table.delete(d); -492 Put put = new Put(ONE_ROW); -493 put.addColumn(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc")); -494 put.addColumn(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L)); -495 table.put(put); -496 -497 Increment inc = new Increment(ONE_ROW); -498 inc.addColumn(BYTES_FAMILY, QUAL2, 1); -499 inc.addColumn(BYTES_FAMILY, QUAL3, 1); -500 -501 Append a = new Append(ONE_ROW); -502 a.addColumn(BYTES_FAMILY, QUAL1, Bytes.toBytes("def")); -503 a.addColumn(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz")); -504 List<Row> actions = new ArrayList<>(); -505 actions.add(inc); -506 actions.add(a); -507 -508 Object[] multiRes = new Object[actions.size()]; -509 table.batch(actions, multiRes); -510 validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef")); -511 validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz")); -512 validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L)); -513 validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L)); -514 table.close(); -515 } -516 -517 @Test(timeout=300000) -518 public void testNonceCollision() throws Exception { -519 LOG.info("test=testNonceCollision"); -520 final Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); -521 Table table = connection.getTable(TEST_TABLE); -522 Put put = new Put(ONE_ROW); -523 put.addColumn(BYTES_FAMILY, QUALIFIER, Bytes.toBytes(0L)); -524 -525 // Replace nonce manager with the one that returns each nonce twice. -526 NonceGenerator cnm = new NonceGenerator() { -527 -528 private final PerClientRandomNonceGenerator delegate = PerClientRandomNonceGenerator.get(); +251 // row1 and row2 should be in the same region. +252 +253 Object [] r = new Object[actions.size()]; +254 try { +255 table.batch(actions, r); +256 fail(); +257 } catch (RetriesExhaustedWithDetailsException ex) { +258 LOG.debug(ex.toString(), ex); +259 // good! +260 assertFalse(ex.mayHaveClusterIssues()); +261 } +262 assertEquals(2, r.length); +263 assertTrue(r[0] instanceof Throwable); +264 assertTrue(r[1] instanceof Result); +265 table.close(); +266 } +267 +268 @Test (timeout=300000) +269 public void testFlushCommitsNoAbort() throws Exception { +270 LOG.info("test=testFlushCommitsNoAbort"); +271 doTestFlushCommits(false); +272 } +273 +274 /** +275 * Only run one Multi test with a forced RegionServer abort. Otherwise, the +276 * unit tests will take an unnecessarily long time to run. +277 * +278 * @throws Exception +279 */ +280 @Test (timeout=360000) +281 public void testFlushCommitsWithAbort() throws Exception { +282 LOG.info("test=testFlushCommitsWithAbort"); +283 doTestFlushCommits(true); +284 } +285 +286 /** +287 * Set table auto flush to false and test flushing commits +288 * @param doAbort true if abort one regionserver in the testing +289 * @throws Exception +290 */ +291 private void doTestFlushCommits(boolean doAbort) throws Exception { +292 // Load the data +293 LOG.info("get new table"); +294 Table table = UTIL.getConnection().getTable(TEST_TABLE); +295 +296 LOG.info("constructPutRequests"); +297 List<Put> puts = constructPutRequests(); +298 table.put(puts); +299 LOG.info("puts"); +300 final int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads() +301 .size(); +302 assert liveRScount > 0; +303 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster() +304 .getLiveRegionServerThreads().get(0); +305 if (doAbort) { +306 liveRS.getRegionServer().abort("Aborting for tests", +307 new Exception("doTestFlushCommits")); +308 // If we wait for no regions being online after we abort the server, we +309 // could ensure the master has re-assigned the regions on killed server +310 // after writing successfully. It means the server we aborted is dead +311 // and detected by matser +312 while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) { +313 Thread.sleep(100); +314 } +315 // try putting more keys after the abort. same key/qual... just validating +316 // no exceptions thrown +317 puts = constructPutRequests(); +318 table.put(puts); +319 } +320 +321 LOG.info("validating loaded data"); +322 validateLoadedData(table); +323 +324 // Validate server and region count +325 List<JVMClusterUtil.RegionServerThread> liveRSs = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); +326 int count = 0; +327 for (JVMClusterUtil.RegionServerThread t: liveRSs) { +328 count++; +329 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer()); +330 } +331 LOG.info("Count=" + count); +332 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort, +333 (doAbort ? (liveRScount - 1) : liveRScount), count); +334 if (doAbort) { +335 UTIL.getMiniHBaseCluster().waitOnRegionServer(0); +336 UTIL.waitFor(15 * 1000, new Waiter.Predicate<Exception>() { +337 @Override +338 public boolean evaluate() throws Exception { +339 // Master is also a regionserver, so the count is liveRScount +340 return UTIL.getMiniHBaseCluster().getMaster() +341 .getClusterMetrics().getLiveServerMetrics().size() == liveRScount; +342 } +343 }); +344 UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition()); +345 } +346 +347 table.close(); +348 LOG.info("done"); +349 } +350 +351 @Test (timeout=300000) +352 public void testBatchWithPut() throws Exception { +353 LOG.info("test=testBatchWithPut"); +354 Table table = CONNECTION.getTable(TEST_TABLE); +355 // put multiple rows using a batch +356 List<Put> puts = constructPutRequests(); +357 +358 Object[] results = new Object[puts.size()]; +359 table.batch(puts, results); +360 validateSizeAndEmpty(results, KEYS.length); +361 +362 if (true) { +363 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size(); +364 assert liveRScount > 0; +365 JVMClusterUtil.RegionServerThread liveRS = +366 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0); +367 liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut")); +368 puts = constructPutRequests(); +369 try { +370 results = new Object[puts.size()]; +371 table.batch(puts, results); +372 } catch (RetriesExhaustedWithDetailsException ree) { +373 LOG.info(ree.getExhaustiveDescription()); +374 table.close(); +375 throw ree; +376 } +377 validateSizeAndEmpty(results, KEYS.length); +378 } +379 +380 validateLoadedData(table); +381 table.close(); +382 } +383 +384 @Test(timeout=300000) +385 public void testBatchWithDelete() throws Exception { +386 LOG.info("test=testBatchWithDelete"); +387 Table table = UTIL.getConnection().getTable(TEST_TABLE); +388 +389 // Load some data +390 List<Put> puts = constructPutRequests(); +391 Object[] results = new Object[puts.size()]; +392 table.batch(puts, results); +393 validateSizeAndEmpty(results, KEYS.length); +394 +395 // Deletes +396 List<Row> deletes = new ArrayList<>(); +397 for (int i = 0; i < KEYS.length; i++) { +398 Delete delete = new Delete(KEYS[i]); +399 delete.addFamily(BYTES_FAMILY); +400 deletes.add(delete); +401 } +402 results= new Object[deletes.size()]; +403 table.batch(deletes, results); +404 validateSizeAndEmpty(results, KEYS.length); +405 +406 // Get to make sure ... +407 for (byte[] k : KEYS) { +408 Get get = new Get(k); +409 get.addColumn(BYTES_FAMILY, QUALIFIER); +410 Assert.assertFalse(table.exists(get)); +411 } +412 table.close(); +413 } +414 +415 @Test(timeout=300000) +416 public void testHTableDeleteWithList() throws Exception { +417 LOG.info("test=testHTableDeleteWithList"); +418 Table table = UTIL.getConnection().getTable(TEST_TABLE); +419 +420 // Load some data +421 List<Put> puts = constructPutRequests(); +422 Object[] results = new Object[puts.size()]; +423 table.batch(puts, results); +424 validateSizeAndEmpty(results, KEYS.length); +425 +426 // Deletes +427 ArrayList<Delete> deletes = new ArrayList<>(); +428 for (int i = 0; i < KEYS.length; i++) { +429 Delete delete = new Delete(KEYS[i]); +430 delete.addFamily(BYTES_FAMILY); +431 deletes.add(delete); +432 } +433 table.delete(deletes); +434 Assert.assertTrue(deletes.isEmpty()); +435 +436 // Get to make sure ... +437 for (byte[] k : KEYS) { +438 Get get = new Get(k); +439 get.addColumn(BYTES_FAMILY, QUALIFIER); +440 Assert.assertFalse(table.exists(get)); +441 } +442 table.close(); +443 } +444 +445 @Test(timeout=300000) +446 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception { +447 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut"); +448 Table table = UTIL.getConnection().getTable(TEST_TABLE); +449 +450 List<Row> puts = new ArrayList<>(); +451 for (int i = 0; i < 100; i++) { +452 Put put = new Put(ONE_ROW); +453 byte[] qual = Bytes.toBytes("column" + i); +454 put.addColumn(BYTES_FAMILY, qual, VALUE); +455 puts.add(put); +456 } +457 Object[] results = new Object[puts.size()]; +458 table.batch(puts, results); +459 +460 // validate +461 validateSizeAndEmpty(results, 100); +462 +463 // get the data back and validate that it is correct +464 List<Row> gets = new ArrayList<>(); +465 for (int i = 0; i < 100; i++) { +466 Get get = new Get(ONE_ROW); +467 byte[] qual = Bytes.toBytes("column" + i); +468 get.addColumn(BYTES_FAMILY, qual); +469 gets.add(get); +470 } +471 +472 Object[] multiRes = new Object[gets.size()]; +473 table.batch(gets, multiRes); +474 +475 int idx = 0; +476 for (Object r : multiRes) { +477 byte[] qual = Bytes.toBytes("column" + idx); +478 validateResult(r, qual, VALUE); +479 idx++; +480 } +481 table.close(); +482 } +483 +484 @Test(timeout=300000) +485 public void testBatchWithIncrementAndAppend() throws Exception { +486 LOG.info("test=testBatchWithIncrementAndAppend"); +487 final byte[] QUAL1 = Bytes.toBytes("qual1"); +488 final byte[] QUAL2 = Bytes.toBytes("qual2"); +489 final byte[] QUAL3 = Bytes.toBytes("qual3"); +490 final byte[] QUAL4 = Bytes.toBytes("qual4"); +491 Table table = UTIL.getConnection().getTable(TEST_TABLE); +492 Delete d = new Delete(ONE_ROW); +493 table.delete(d); +494 Put put = new Put(ONE_ROW); +495 put.addColumn(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc")); +496 put.addColumn(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L)); +497 table.put(put); +498 +499 Increment inc = new Increment(ONE_ROW); +500 inc.addColumn(BYTES_FAMILY, QUAL2, 1);