Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6F780200C7D for ; Tue, 16 May 2017 16:59:21 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6DDFC160BDC; Tue, 16 May 2017 14:59:21 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 43903160BCF for ; Tue, 16 May 2017 16:59:18 +0200 (CEST) Received: (qmail 55612 invoked by uid 500); 16 May 2017 14:59:17 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 55199 invoked by uid 99); 16 May 2017 14:59:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 May 2017 14:59:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 759B8E115D; Tue, 16 May 2017 14:59:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 16 May 2017 14:59:18 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [04/42] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Tue, 16 May 2017 14:59:21 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/de34004c/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide.ExceptionInReseekRegionObserver.MyStoreScanner.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide.ExceptionInReseekRegionObserver.MyStoreScanner.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide.ExceptionInReseekRegionObserver.MyStoreScanner.html index aeab214..7d32ed2 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide.ExceptionInReseekRegionObserver.MyStoreScanner.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide.ExceptionInReseekRegionObserver.MyStoreScanner.html @@ -107,6199 +107,6236 @@ 099import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; 100import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest; 101import org.apache.hadoop.hbase.regionserver.DelegatingKeyValueScanner; -102import org.apache.hadoop.hbase.regionserver.HRegionServer; -103import org.apache.hadoop.hbase.regionserver.KeyValueScanner; -104import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; -105import org.apache.hadoop.hbase.regionserver.Region; -106import org.apache.hadoop.hbase.regionserver.ScanInfo; -107import org.apache.hadoop.hbase.regionserver.Store; -108import org.apache.hadoop.hbase.regionserver.StoreScanner; -109import org.apache.hadoop.hbase.testclassification.ClientTests; -110import org.apache.hadoop.hbase.testclassification.LargeTests; -111import org.apache.hadoop.hbase.util.Bytes; -112import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -113import org.apache.hadoop.hbase.util.Pair; -114import org.junit.After; -115import org.junit.AfterClass; -116import org.junit.Before; -117import org.junit.BeforeClass; -118import org.junit.Ignore; -119import org.junit.Rule; -120import org.junit.Test; -121import org.junit.experimental.categories.Category; -122import org.junit.rules.TestName; -123 -124/** -125 * Run tests that use the HBase clients; {@link Table}. -126 * Sets up the HBase mini cluster once at start and runs through all client tests. -127 * Each creates a table named for the method and does its stuff against that. -128 */ -129@Category({LargeTests.class, ClientTests.class}) -130@SuppressWarnings ("deprecation") -131public class TestFromClientSide { -132 // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide. -133 private static final Log LOG = LogFactory.getLog(TestFromClientSide.class); -134 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); -135 private static byte [] ROW = Bytes.toBytes("testRow"); -136 private static byte [] FAMILY = Bytes.toBytes("testFamily"); -137 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); -138 private static byte [] VALUE = Bytes.toBytes("testValue"); -139 protected static int SLAVES = 3; -140 -141 @Rule -142 public TestName name = new TestName(); -143 -144 /** -145 * @throws java.lang.Exception -146 */ -147 @BeforeClass -148 public static void setUpBeforeClass() throws Exception { -149 // Uncomment the following lines if more verbosity is needed for -150 // debugging (see HBASE-12285 for details). -151 //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); -152 //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); -153 //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); -154 Configuration conf = TEST_UTIL.getConfiguration(); -155 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, -156 MultiRowMutationEndpoint.class.getName()); -157 conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests -158 conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 6000000); -159 // We need more than one region server in this test -160 TEST_UTIL.startMiniCluster(SLAVES); -161 } -162 -163 /** -164 * @throws java.lang.Exception -165 */ -166 @AfterClass -167 public static void tearDownAfterClass() throws Exception { -168 TEST_UTIL.shutdownMiniCluster(); -169 } -170 -171 /** -172 * @throws java.lang.Exception -173 */ -174 @Before -175 public void setUp() throws Exception { -176 // Nothing to do. -177 } -178 -179 /** -180 * @throws java.lang.Exception -181 */ -182 @After -183 public void tearDown() throws Exception { -184 // Nothing to do. -185 } -186 -187 /** -188 * Test append result when there are duplicate rpc request. -189 */ -190 @Test -191 public void testDuplicateAppend() throws Exception { -192 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(name.getMethodName()); -193 Map<String, String> kvs = new HashMap<>(); -194 kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000"); -195 hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs); -196 TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close(); -197 -198 Configuration c = new Configuration(TEST_UTIL.getConfiguration()); -199 c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50); -200 // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call -201 c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); -202 -203 Connection connection = ConnectionFactory.createConnection(c); -204 Table t = connection.getTable(TableName.valueOf(name.getMethodName())); -205 if (t instanceof HTable) { -206 HTable table = (HTable) t; -207 table.setOperationTimeout(3 * 1000); -208 -209 try { -210 Append append = new Append(ROW); -211 append.add(TEST_UTIL.fam1, QUALIFIER, VALUE); -212 Result result = table.append(append); -213 -214 // Verify expected result -215 Cell[] cells = result.rawCells(); -216 assertEquals(1, cells.length); -217 assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE); -218 -219 // Verify expected result again -220 Result readResult = table.get(new Get(ROW)); -221 cells = readResult.rawCells(); -222 assertEquals(1, cells.length); -223 assertKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, VALUE); -224 } finally { -225 table.close(); -226 connection.close(); -227 } -228 } -229 } -230 -231 /** -232 * Basic client side validation of HBASE-4536 -233 */ -234 @Test -235 public void testKeepDeletedCells() throws Exception { -236 final TableName tableName = TableName.valueOf(name.getMethodName()); -237 final byte[] FAMILY = Bytes.toBytes("family"); -238 final byte[] C0 = Bytes.toBytes("c0"); -239 -240 final byte[] T1 = Bytes.toBytes("T1"); -241 final byte[] T2 = Bytes.toBytes("T2"); -242 final byte[] T3 = Bytes.toBytes("T3"); -243 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY) -244 .setKeepDeletedCells(KeepDeletedCells.TRUE) -245 .setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE) -246 .setMaxVersions(3); -247 -248 HTableDescriptor desc = new HTableDescriptor(tableName); -249 desc.addFamily(hcd); -250 TEST_UTIL.getAdmin().createTable(desc); -251 Table h = TEST_UTIL.getConnection().getTable(tableName); -252 -253 long ts = System.currentTimeMillis(); -254 Put p = new Put(T1, ts); -255 p.addColumn(FAMILY, C0, T1); -256 h.put(p); -257 p = new Put(T1, ts+2); -258 p.addColumn(FAMILY, C0, T2); -259 h.put(p); -260 p = new Put(T1, ts+4); -261 p.addColumn(FAMILY, C0, T3); -262 h.put(p); -263 -264 Delete d = new Delete(T1, ts+3); -265 h.delete(d); -266 -267 d = new Delete(T1, ts+3); -268 d.addColumns(FAMILY, C0, ts+3); -269 h.delete(d); -270 -271 Get g = new Get(T1); -272 // does *not* include the delete -273 g.setTimeRange(0, ts+3); -274 Result r = h.get(g); -275 assertArrayEquals(T2, r.getValue(FAMILY, C0)); -276 -277 Scan s = new Scan(T1); -278 s.setTimeRange(0, ts+3); -279 s.setMaxVersions(); -280 ResultScanner scanner = h.getScanner(s); -281 Cell[] kvs = scanner.next().rawCells(); -282 assertArrayEquals(T2, CellUtil.cloneValue(kvs[0])); -283 assertArrayEquals(T1, CellUtil.cloneValue(kvs[1])); -284 scanner.close(); -285 -286 s = new Scan(T1); -287 s.setRaw(true); -288 s.setMaxVersions(); -289 scanner = h.getScanner(s); -290 kvs = scanner.next().rawCells(); -291 assertTrue(CellUtil.isDeleteFamily(kvs[0])); -292 assertArrayEquals(T3, CellUtil.cloneValue(kvs[1])); -293 assertTrue(CellUtil.isDelete(kvs[2])); -294 assertArrayEquals(T2, CellUtil.cloneValue(kvs[3])); -295 assertArrayEquals(T1, CellUtil.cloneValue(kvs[4])); -296 scanner.close(); -297 h.close(); -298 } -299 -300 /** -301 * Basic client side validation of HBASE-10118 -302 */ -303 @Test -304 public void testPurgeFutureDeletes() throws Exception { -305 final TableName tableName = TableName.valueOf(name.getMethodName()); -306 final byte[] ROW = Bytes.toBytes("row"); -307 final byte[] FAMILY = Bytes.toBytes("family"); -308 final byte[] COLUMN = Bytes.toBytes("column"); -309 final byte[] VALUE = Bytes.toBytes("value"); -310 -311 Table table = TEST_UTIL.createTable(tableName, FAMILY); -312 -313 // future timestamp -314 long ts = System.currentTimeMillis() * 2; -315 Put put = new Put(ROW, ts); -316 put.addColumn(FAMILY, COLUMN, VALUE); -317 table.put(put); -318 -319 Get get = new Get(ROW); -320 Result result = table.get(get); -321 assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); -322 -323 Delete del = new Delete(ROW); -324 del.addColumn(FAMILY, COLUMN, ts); -325 table.delete(del); -326 -327 get = new Get(ROW); -328 result = table.get(get); -329 assertNull(result.getValue(FAMILY, COLUMN)); -330 -331 // major compaction, purged future deletes -332 TEST_UTIL.getAdmin().flush(tableName); -333 TEST_UTIL.getAdmin().majorCompact(tableName); -334 -335 // waiting for the major compaction to complete -336 TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() { -337 @Override -338 public boolean evaluate() throws IOException { -339 return TEST_UTIL.getAdmin().getCompactionState(tableName) == -340 CompactionState.NONE; -341 } -342 }); -343 -344 put = new Put(ROW, ts); -345 put.addColumn(FAMILY, COLUMN, VALUE); -346 table.put(put); -347 -348 get = new Get(ROW); -349 result = table.get(get); -350 assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); -351 -352 table.close(); -353 } -354 -355 /** -356 * Verifies that getConfiguration returns the same Configuration object used -357 * to create the HTable instance. -358 */ -359 @Test -360 public void testGetConfiguration() throws Exception { -361 final TableName tableName = TableName.valueOf(name.getMethodName()); -362 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; -363 Configuration conf = TEST_UTIL.getConfiguration(); -364 Table table = TEST_UTIL.createTable(tableName, FAMILIES); -365 assertSame(conf, table.getConfiguration()); -366 } -367 -368 /** -369 * Test from client side of an involved filter against a multi family that -370 * involves deletes. -371 * -372 * @throws Exception -373 */ -374 @Test -375 public void testWeirdCacheBehaviour() throws Exception { -376 final TableName tableName = TableName.valueOf(name.getMethodName()); -377 byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), -378 Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), -379 Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; -380 Table ht = TEST_UTIL.createTable(tableName, FAMILIES); -381 String value = "this is the value"; -382 String value2 = "this is some other value"; -383 String keyPrefix1 = UUID.randomUUID().toString(); -384 String keyPrefix2 = UUID.randomUUID().toString(); -385 String keyPrefix3 = UUID.randomUUID().toString(); -386 putRows(ht, 3, value, keyPrefix1); -387 putRows(ht, 3, value, keyPrefix2); -388 putRows(ht, 3, value, keyPrefix3); -389 putRows(ht, 3, value2, keyPrefix1); -390 putRows(ht, 3, value2, keyPrefix2); -391 putRows(ht, 3, value2, keyPrefix3); -392 Table table = TEST_UTIL.getConnection().getTable(tableName); -393 System.out.println("Checking values for key: " + keyPrefix1); -394 assertEquals("Got back incorrect number of rows from scan", 3, -395 getNumberOfRows(keyPrefix1, value2, table)); -396 System.out.println("Checking values for key: " + keyPrefix2); -397 assertEquals("Got back incorrect number of rows from scan", 3, -398 getNumberOfRows(keyPrefix2, value2, table)); -399 System.out.println("Checking values for key: " + keyPrefix3); -400 assertEquals("Got back incorrect number of rows from scan", 3, -401 getNumberOfRows(keyPrefix3, value2, table)); -402 deleteColumns(ht, value2, keyPrefix1); -403 deleteColumns(ht, value2, keyPrefix2); -404 deleteColumns(ht, value2, keyPrefix3); -405 System.out.println("Starting important checks....."); -406 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, -407 0, getNumberOfRows(keyPrefix1, value2, table)); -408 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, -409 0, getNumberOfRows(keyPrefix2, value2, table)); -410 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, -411 0, getNumberOfRows(keyPrefix3, value2, table)); -412 } -413 -414 private void deleteColumns(Table ht, String value, String keyPrefix) -415 throws IOException { -416 ResultScanner scanner = buildScanner(keyPrefix, value, ht); -417 Iterator<Result> it = scanner.iterator(); -418 int count = 0; -419 while (it.hasNext()) { -420 Result result = it.next(); -421 Delete delete = new Delete(result.getRow()); -422 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); -423 ht.delete(delete); -424 count++; -425 } -426 assertEquals("Did not perform correct number of deletes", 3, count); -427 } -428 -429 private int getNumberOfRows(String keyPrefix, String value, Table ht) -430 throws Exception { -431 ResultScanner resultScanner = buildScanner(keyPrefix, value, ht); -432 Iterator<Result> scanner = resultScanner.iterator(); -433 int numberOfResults = 0; -434 while (scanner.hasNext()) { -435 Result result = scanner.next(); -436 System.out.println("Got back key: " + Bytes.toString(result.getRow())); -437 for (Cell kv : result.rawCells()) { -438 System.out.println("kv=" + kv.toString() + ", " -439 + Bytes.toString(CellUtil.cloneValue(kv))); -440 } -441 numberOfResults++; -442 } -443 return numberOfResults; -444 } -445 -446 private ResultScanner buildScanner(String keyPrefix, String value, Table ht) -447 throws IOException { -448 // OurFilterList allFilters = new OurFilterList(); -449 FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); -450 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); -451 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes -452 .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes -453 .toBytes(value)); -454 filter.setFilterIfMissing(true); -455 allFilters.addFilter(filter); -456 -457 // allFilters.addFilter(new -458 // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), -459 // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); -460 -461 Scan scan = new Scan(); -462 scan.addFamily(Bytes.toBytes("trans-blob")); -463 scan.addFamily(Bytes.toBytes("trans-type")); -464 scan.addFamily(Bytes.toBytes("trans-date")); -465 scan.addFamily(Bytes.toBytes("trans-tags")); -466 scan.addFamily(Bytes.toBytes("trans-group")); -467 scan.setFilter(allFilters); -468 -469 return ht.getScanner(scan); -470 } -471 -472 private void putRows(Table ht, int numRows, String value, String key) -473 throws IOException { -474 for (int i = 0; i < numRows; i++) { -475 String row = key + "_" + UUID.randomUUID().toString(); -476 System.out.println(String.format("Saving row: %s, with value %s", row, -477 value)); -478 Put put = new Put(Bytes.toBytes(row)); -479 put.setDurability(Durability.SKIP_WAL); -480 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes -481 .toBytes("value for blob")); -482 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); -483 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes -484 .toBytes("20090921010101999")); -485 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes -486 .toBytes(value)); -487 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes -488 .toBytes("adhocTransactionGroupId")); -489 ht.put(put); -490 } -491 } -492 -493 /** -494 * Test filters when multiple regions. It does counts. Needs eye-balling of -495 * logs to ensure that we're not scanning more regions that we're supposed to. -496 * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package. -497 * @throws IOException -498 * @throws InterruptedException -499 */ -500 @Test -501 public void testFilterAcrossMultipleRegions() -502 throws IOException, InterruptedException { -503 final TableName tableName = TableName.valueOf(name.getMethodName()); -504 Table t = TEST_UTIL.createTable(tableName, FAMILY); -505 int rowCount = TEST_UTIL.loadTable(t, FAMILY, false); -506 assertRowCount(t, rowCount); -507 // Split the table. Should split on a reasonable key; 'lqj' -508 List<HRegionLocation> regions = splitTable(t); -509 assertRowCount(t, rowCount); -510 // Get end key of first region. -511 byte [] endKey = regions.get(0).getRegionInfo().getEndKey(); -512 // Count rows with a filter that stops us before passed 'endKey'. -513 // Should be count of rows in first region. -514 int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey)); -515 assertTrue(endKeyCount < rowCount); -516 -517 // How do I know I did not got to second region? Thats tough. Can't really -518 // do that in client-side region test. I verified by tracing in debugger. -519 // I changed the messages that come out when set to DEBUG so should see -520 // when scanner is done. Says "Finished with scanning..." with region name. -521 // Check that its finished in right region. -522 -523 // New test. Make it so scan goes into next region by one and then two. -524 // Make sure count comes out right. -525 byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)}; -526 int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); -527 assertEquals(endKeyCount + 1, plusOneCount); -528 key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)}; -529 int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); -530 assertEquals(endKeyCount + 2, plusTwoCount); -531 -532 // New test. Make it so I scan one less than endkey. -533 key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)}; -534 int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); -535 assertEquals(endKeyCount - 1, minusOneCount); -536 // For above test... study logs. Make sure we do "Finished with scanning.." -537 // in first region and that we do not fall into the next region. -538 -539 key = new byte [] {'a', 'a', 'a'}; -540 int countBBB = TEST_UTIL.countRows(t, -541 createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL)); -542 assertEquals(1, countBBB); -543 -544 int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null, -545 CompareFilter.CompareOp.GREATER_OR_EQUAL)); -546 // Because started at start of table. -547 assertEquals(0, countGreater); -548 countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey, -549 CompareFilter.CompareOp.GREATER_OR_EQUAL)); -550 assertEquals(rowCount - endKeyCount, countGreater); -551 } -552 -553 /** -554 * This is a coprocessor to inject a test failure so that a store scanner.reseek() call will -555 * fail with an IOException() on the first call. -556 */ -557 public static class ExceptionInReseekRegionObserver implements RegionObserver { -558 static AtomicLong reqCount = new AtomicLong(0); -559 static AtomicBoolean isDoNotRetry = new AtomicBoolean(false); // whether to throw DNRIOE -560 static AtomicBoolean throwOnce = new AtomicBoolean(true); // whether to only throw once -561 -562 static void reset() { -563 reqCount.set(0); -564 isDoNotRetry.set(false); -565 throwOnce.set(true); -566 } -567 -568 class MyStoreScanner extends StoreScanner { -569 public MyStoreScanner(Store store, ScanInfo scanInfo, Scan scan, NavigableSet<byte[]> columns, -570 long readPt) throws IOException { -571 super(store, scanInfo, scan, columns, readPt); -572 } -573 -574 @Override -575 protected List<KeyValueScanner> selectScannersFrom( -576 List<? extends KeyValueScanner> allScanners) { -577 List<KeyValueScanner> scanners = super.selectScannersFrom(allScanners); -578 List<KeyValueScanner> newScanners = new ArrayList<>(scanners.size()); -579 for (KeyValueScanner scanner : scanners) { -580 newScanners.add(new DelegatingKeyValueScanner(scanner) { -581 @Override -582 public boolean reseek(Cell key) throws IOException { -583 reqCount.incrementAndGet(); -584 if (!throwOnce.get()|| reqCount.get() == 1) { -585 if (isDoNotRetry.get()) { -586 throw new DoNotRetryIOException("Injected exception"); -587 } else { -588 throw new IOException("Injected exception"); -589 } -590 } -591 return super.reseek(key); -592 } -593 }); -594 } -595 return newScanners; -596 } -597 } -598 -599 @Override -600 public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, -601 Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s, -602 final long readPt) throws IOException { -603 return new MyStoreScanner(store, store.getScanInfo(), scan, targetCols, readPt); -604 } -605 } -606 -607 /** -608 * Tests the case where a Scan can throw an IOException in the middle of the seek / reseek -609 * leaving the server side RegionScanner to be in dirty state. The client has to ensure that the -610 * ClientScanner does not get an exception and also sees all the data. -611 * @throws IOException -612 * @throws InterruptedException -613 */ -614 @Test -615 public void testClientScannerIsResetWhenScanThrowsIOException() -616 throws IOException, InterruptedException { -617 TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true); -618 final TableName tableName = TableName.valueOf(name.getMethodName()); -619 -620 HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY); -621 htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName()); -622 TEST_UTIL.getAdmin().createTable(htd); -623 ExceptionInReseekRegionObserver.reset(); -624 ExceptionInReseekRegionObserver.throwOnce.set(true); // throw exceptions only once -625 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { -626 int rowCount = TEST_UTIL.loadTable(t, FAMILY, false); -627 TEST_UTIL.getAdmin().flush(tableName); -628 int actualRowCount = TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY)); -629 assertEquals(rowCount, actualRowCount); -630 } -631 assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0); -632 } -633 -634 /** -635 * Tests the case where a coprocessor throws a DoNotRetryIOException in the scan. The expectation -636 * is that the exception will bubble up to the client scanner instead of being retried. -637 */ -638 @Test (timeout = 180000) -639 public void testScannerThrowsExceptionWhenCoprocessorThrowsDNRIOE() -640 throws IOException, InterruptedException { -641 TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true); -642 final TableName tableName = TableName.valueOf(name.getMethodName()); -643 -644 HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY); -645 htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName()); -646 TEST_UTIL.getAdmin().createTable(htd); -647 ExceptionInReseekRegionObserver.reset(); -648 ExceptionInReseekRegionObserver.isDoNotRetry.set(true); -649 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { -650 TEST_UTIL.loadTable(t, FAMILY, false); -651 TEST_UTIL.getAdmin().flush(tableName); -652 TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY)); -653 fail("Should have thrown an exception"); -654 } catch (DoNotRetryIOException expected) { -655 // expected -656 } -657 assertTrue(ExceptionInReseekRegionObserver.reqCount.get() > 0); -658 } -659 -660 /** -661 * Tests the case where a coprocessor throws a regular IOException in the scan. The expectation -662 * is that the we will keep on retrying, but fail after the retries are exhausted instead of -663 * retrying indefinitely. -664 */ -665 @Test (timeout = 180000) -666 public void testScannerFailsAfterRetriesWhenCoprocessorThrowsIOE() -667 throws IOException, InterruptedException { -668 TEST_UTIL.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true); -669 final TableName tableName = TableName.valueOf(name.getMethodName()); -670 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); -671 HTableDescriptor htd = TEST_UTIL.createTableDescriptor(tableName, FAMILY); -672 htd.addCoprocessor(ExceptionInReseekRegionObserver.class.getName()); -673 TEST_UTIL.getAdmin().createTable(htd); -674 ExceptionInReseekRegionObserver.reset(); -675 ExceptionInReseekRegionObserver.throwOnce.set(false); // throw exceptions in every retry -676 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { -677 TEST_UTIL.loadTable(t, FAMILY, false); -678 TEST_UTIL.getAdmin().flush(tableName); -679 TEST_UTIL.countRows(t, new Scan().addColumn(FAMILY, FAMILY)); -680 fail("Should have thrown an exception"); -681 } catch (DoNotRetryIOException expected) { -682 assertTrue(expected instanceof ScannerResetException); -683 // expected -684 } -685 assertTrue(ExceptionInReseekRegionObserver.reqCount.get() >= 3); -686 } -687 -688 /* -689 * @param key -690 * @return Scan with RowFilter that does LESS than passed key. -691 */ -692 private Scan createScanWithRowFilter(final byte [] key) { -693 return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS); -694 } -695 -696 /* -697 * @param key -698 * @param op -699 * @param startRow -700 * @return Scan with RowFilter that does CompareOp op on passed key. -701 */ -702 private Scan createScanWithRowFilter(final byte [] key, -703 final byte [] startRow, CompareFilter.CompareOp op) { -704 // Make sure key is of some substance... non-null and > than first key. -705 assertTrue(key != null && key.length > 0 && -706 Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0); -707 LOG.info("Key=" + Bytes.toString(key)); -708 Scan s = startRow == null? new Scan(): new Scan(startRow); -709 Filter f = new RowFilter(op, new BinaryComparator(key)); -710 f = new WhileMatchFilter(f); -711 s.setFilter(f); -712 return s; -713 } -714 -715 private void assertRowCount(final Table t, final int expected) -716 throws IOException { -717 assertEquals(expected, TEST_UTIL.countRows(t, new Scan())); -718 } -719 -720 /* -721 * Split table into multiple regions. -722 * @param t Table to split. -723 * @return Map of regions to servers. -724 * @throws IOException -725 */ -726 private List<HRegionLocation> splitTable(final Table t) -727 throws IOException, InterruptedException { -728 // Split this table in two. -729 Admin admin = TEST_UTIL.getAdmin(); -730 admin.split(t.getName()); -731 admin.close(); -732 List<HRegionLocation> regions = waitOnSplit(t); -733 assertTrue(regions.size() > 1); -734 return regions; -735 } -736 -737 /* -738 * Wait on table split. May return because we waited long enough on the split -739 * and it didn't happen. Caller should check. -740 * @param t -741 * @return Map of table regions; caller needs to check table actually split. -742 */ -743 private List<HRegionLocation> waitOnSplit(final Table t) -744 throws IOException { -745 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) { -746 List<HRegionLocation> regions = locator.getAllRegionLocations(); -747 int originalCount = regions.size(); -748 for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) { -749 Thread.currentThread(); -750 try { -751 Thread.sleep(1000); -752 } catch (InterruptedException e) { -753 e.printStackTrace(); -754 } -755 regions = locator.getAllRegionLocations(); -756 if (regions.size() > originalCount) -757 break; -758 } -759 return regions; -760 } -761 } -762 -763 @Test -764 public void testSuperSimple() throws Exception { -765 final TableName tableName = TableName.valueOf(name.getMethodName()); -766 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -767 Put put = new Put(ROW); -768 put.addColumn(FAMILY, QUALIFIER, VALUE); -769 ht.put(put); -770 Scan scan = new Scan(); -771 scan.addColumn(FAMILY, tableName.toBytes()); -772 ResultScanner scanner = ht.getScanner(scan); -773 Result result = scanner.next(); -774 assertTrue("Expected null result", result == null); -775 scanner.close(); -776 } -777 -778 @Test -779 public void testMaxKeyValueSize() throws Exception { -780 final TableName tableName = TableName.valueOf(name.getMethodName()); -781 Configuration conf = TEST_UTIL.getConfiguration(); -782 String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY); -783 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -784 byte[] value = new byte[4 * 1024 * 1024]; -785 Put put = new Put(ROW); -786 put.addColumn(FAMILY, QUALIFIER, value); -787 ht.put(put); -788 try { -789 TEST_UTIL.getConfiguration().setInt( -790 ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024); -791 // Create new table so we pick up the change in Configuration. -792 try (Connection connection = -793 ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { -794 try (Table t = connection.getTable(TableName.valueOf(FAMILY))) { -795 put = new Put(ROW); -796 put.addColumn(FAMILY, QUALIFIER, value); -797 t.put(put); -798 } -799 } -800 fail("Inserting a too large KeyValue worked, should throw exception"); -801 } catch(Exception e) {} -802 conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize); -803 } -804 -805 @Test -806 public void testFilters() throws Exception { -807 final TableName tableName = TableName.valueOf(name.getMethodName()); -808 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -809 byte [][] ROWS = makeN(ROW, 10); -810 byte [][] QUALIFIERS = { -811 Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"), -812 Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"), -813 Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"), -814 Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"), -815 Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") -816 }; -817 for(int i=0;i<10;i++) { -818 Put put = new Put(ROWS[i]); -819 put.setDurability(Durability.SKIP_WAL); -820 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); -821 ht.put(put); -822 } -823 Scan scan = new Scan(); -824 scan.addFamily(FAMILY); -825 Filter filter = new QualifierFilter(CompareOp.EQUAL, -826 new RegexStringComparator("col[1-5]")); -827 scan.setFilter(filter); -828 ResultScanner scanner = ht.getScanner(scan); -829 int expectedIndex = 1; -830 for(Result result : ht.getScanner(scan)) { -831 assertEquals(result.size(), 1); -832 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex])); -833 assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]), -834 QUALIFIERS[expectedIndex])); -835 expectedIndex++; -836 } -837 assertEquals(expectedIndex, 6); -838 scanner.close(); -839 } -840 -841 @Test -842 public void testFilterWithLongCompartor() throws Exception { -843 final TableName tableName = TableName.valueOf(name.getMethodName()); -844 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -845 byte [][] ROWS = makeN(ROW, 10); -846 byte [][] values = new byte[10][]; -847 for (int i = 0; i < 10; i ++) { -848 values[i] = Bytes.toBytes(100L * i); -849 } -850 for(int i = 0; i < 10; i ++) { -851 Put put = new Put(ROWS[i]); -852 put.setDurability(Durability.SKIP_WAL); -853 put.addColumn(FAMILY, QUALIFIER, values[i]); -854 ht.put(put); -855 } -856 Scan scan = new Scan(); -857 scan.addFamily(FAMILY); -858 Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER, -859 new LongComparator(500)); -860 scan.setFilter(filter); -861 ResultScanner scanner = ht.getScanner(scan); -862 int expectedIndex = 0; -863 for(Result result : ht.getScanner(scan)) { -864 assertEquals(result.size(), 1); -865 assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500); -866 expectedIndex++; -867 } -868 assertEquals(expectedIndex, 4); -869 scanner.close(); -870} -871 -872 @Test -873 public void testKeyOnlyFilter() throws Exception { -874 final TableName tableName = TableName.valueOf(name.getMethodName()); -875 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -876 byte [][] ROWS = makeN(ROW, 10); -877 byte [][] QUALIFIERS = { -878 Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"), -879 Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"), -880 Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"), -881 Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"), -882 Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") -883 }; -884 for(int i=0;i<10;i++) { -885 Put put = new Put(ROWS[i]); -886 put.setDurability(Durability.SKIP_WAL); -887 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); -888 ht.put(put); -889 } -890 Scan scan = new Scan(); -891 scan.addFamily(FAMILY); -892 Filter filter = new KeyOnlyFilter(true); -893 scan.setFilter(filter); -894 ResultScanner scanner = ht.getScanner(scan); -895 int count = 0; -896 for(Result result : ht.getScanner(scan)) { -897 assertEquals(result.size(), 1); -898 assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT); -899 assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length); -900 count++; -901 } -902 assertEquals(count, 10); -903 scanner.close(); -904 } -905 -906 /** -907 * Test simple table and non-existent row cases. -908 */ -909 @Test -910 public void testSimpleMissing() throws Exception { -911 final TableName tableName = TableName.valueOf(name.getMethodName()); -912 Table ht = TEST_UTIL.createTable(tableName, FAMILY); -913 byte [][] ROWS = makeN(ROW, 4); -914 -915 // Try to get a row on an empty table -916 Get get = new Get(ROWS[0]); -917 Result result = ht.get(get); -918 assertEmptyResult(result); -919 -920 get = new Get(ROWS[0]); -921 get.addFamily(FAMILY); -922 result = ht.get(get); -923 assertEmptyResult(result); -924 -925 get = new Get(ROWS[0]); -926 get.addColumn(FAMILY, QUALIFIER); -927 result = ht.get(get); -928 assertEmptyResult(result); -929 -930 Scan scan = new Scan(); -931 result = getSingleScanResult(ht, scan); -932 assertNullResult(result); -933 +102import org.apache.hadoop.hbase.regionserver.HRegion; +103import org.apache.hadoop.hbase.regionserver.HRegionServer; +104import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +105import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; +106import org.apache.hadoop.hbase.regionserver.Region; +107import org.apache.hadoop.hbase.regionserver.ScanInfo; +108import org.apache.hadoop.hbase.regionserver.Store; +109import org.apache.hadoop.hbase.regionserver.StoreScanner; +110import org.apache.hadoop.hbase.testclassification.ClientTests; +111import org.apache.hadoop.hbase.testclassification.LargeTests; +112import org.apache.hadoop.hbase.util.Bytes; +113import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +114import org.apache.hadoop.hbase.util.Pair; +115import org.junit.After; +116import org.junit.AfterClass; +117import org.junit.Before; +118import org.junit.BeforeClass; +119import org.junit.Ignore; +120import org.junit.Rule; +121import org.junit.Test; +122import org.junit.experimental.categories.Category; +123import org.junit.rules.TestName; +124 +125/** +126 * Run tests that use the HBase clients; {@link Table}. +127 * Sets up the HBase mini cluster once at start and runs through all client tests. +128 * Each creates a table named for the method and does its stuff against that. +129 */ +130@Category({LargeTests.class, ClientTests.class}) +131@SuppressWarnings ("deprecation") +132public class TestFromClientSide { +133 // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide. +134 private static final Log LOG = LogFactory.getLog(TestFromClientSide.class); +135 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); +136 private static byte [] ROW = Bytes.toBytes("testRow"); +137 private static byte [] FAMILY = Bytes.toBytes("testFamily"); +138 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); +139 private static byte [] VALUE = Bytes.toBytes("testValue"); +140 protected static int SLAVES = 3; +141 +142 @Rule