Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8F2CF200CE9 for ; Sat, 19 Aug 2017 22:02:41 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8DB24164D4C; Sat, 19 Aug 2017 20:02:41 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0FF2A164D43 for ; Sat, 19 Aug 2017 22:02:38 +0200 (CEST) Received: (qmail 77862 invoked by uid 500); 19 Aug 2017 20:02:37 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 76077 invoked by uid 99); 19 Aug 2017 20:02:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 19 Aug 2017 20:02:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A0A53F5EF7; Sat, 19 Aug 2017 20:02:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 19 Aug 2017 20:02:36 -0000 Message-Id: In-Reply-To: <1b11da23a7654517a171a4ae5c02a3b3@git.apache.org> References: <1b11da23a7654517a171a4ae5c02a3b3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/30] hbase-site git commit: Published site at . archived-at: Sat, 19 Aug 2017 20:02:41 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/495ddb86/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide3.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide3.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide3.html index 401b413..6bb0bed 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide3.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/client/TestFromClientSide3.html @@ -42,880 +42,981 @@ 034import org.apache.commons.logging.LogFactory; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.hbase.Cell; -037import org.apache.hadoop.hbase.Coprocessor; -038import org.apache.hadoop.hbase.coprocessor.ObserverContext; -039import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -040import org.apache.hadoop.hbase.coprocessor.RegionObserver; -041import org.apache.hadoop.hbase.regionserver.HRegionServer; -042import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; -043import org.apache.hadoop.hbase.TableName; -044import org.apache.hadoop.hbase.regionserver.Region; -045import org.apache.hadoop.hbase.regionserver.RegionScanner; -046import org.apache.hadoop.hbase.HBaseTestingUtility; -047import org.apache.hadoop.hbase.HColumnDescriptor; -048import org.apache.hadoop.hbase.HConstants; -049import org.apache.hadoop.hbase.HRegionLocation; -050import org.apache.hadoop.hbase.HTableDescriptor; -051import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; -052import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -053import org.apache.hadoop.hbase.ipc.ServerRpcController; -054import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; -055import org.apache.hadoop.hbase.regionserver.HRegion; -056import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -057import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -058import org.apache.hadoop.hbase.testclassification.ClientTests; -059import org.apache.hadoop.hbase.testclassification.LargeTests; -060import org.apache.hadoop.hbase.util.Bytes; -061import org.apache.hadoop.hbase.util.Pair; -062import org.junit.After; -063import org.junit.AfterClass; -064import static org.junit.Assert.assertEquals; -065import static org.junit.Assert.assertFalse; -066import static org.junit.Assert.assertNull; -067import static org.junit.Assert.assertTrue; -068import static org.junit.Assert.fail; -069import org.junit.Before; -070import org.junit.BeforeClass; -071import org.junit.Rule; -072import org.junit.Test; -073import org.junit.experimental.categories.Category; -074import org.junit.rules.TestName; -075 -076@Category({LargeTests.class, ClientTests.class}) -077public class TestFromClientSide3 { -078 private static final Log LOG = LogFactory.getLog(TestFromClientSide3.class); -079 private final static HBaseTestingUtility TEST_UTIL -080 = new HBaseTestingUtility(); -081 private static byte[] FAMILY = Bytes.toBytes("testFamily"); -082 private static Random random = new Random(); -083 private static int SLAVES = 3; -084 private static byte [] ROW = Bytes.toBytes("testRow"); -085 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow"); -086 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); -087 private static byte [] VALUE = Bytes.toBytes("testValue"); -088 private final static byte[] COL_QUAL = Bytes.toBytes("f1"); -089 private final static byte[] VAL_BYTES = Bytes.toBytes("v1"); -090 private final static byte[] ROW_BYTES = Bytes.toBytes("r1"); -091 -092 @Rule -093 public TestName name = new TestName(); -094 -095 /** -096 * @throws java.lang.Exception -097 */ -098 @BeforeClass -099 public static void setUpBeforeClass() throws Exception { -100 TEST_UTIL.startMiniCluster(SLAVES); -101 } -102 -103 /** -104 * @throws java.lang.Exception -105 */ -106 @AfterClass -107 public static void tearDownAfterClass() throws Exception { -108 TEST_UTIL.shutdownMiniCluster(); -109 } -110 -111 /** -112 * @throws java.lang.Exception -113 */ -114 @Before -115 public void setUp() throws Exception { -116 // Nothing to do. -117 } -118 -119 /** -120 * @throws java.lang.Exception -121 */ -122 @After -123 public void tearDown() throws Exception { -124 for (HTableDescriptor htd: TEST_UTIL.getAdmin().listTables()) { -125 LOG.info("Tear down, remove table=" + htd.getTableName()); -126 TEST_UTIL.deleteTable(htd.getTableName()); -127 } +037import org.apache.hadoop.hbase.CellUtil; +038import org.apache.hadoop.hbase.Coprocessor; +039import org.apache.hadoop.hbase.coprocessor.ObserverContext; +040import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +041import org.apache.hadoop.hbase.coprocessor.RegionObserver; +042import org.apache.hadoop.hbase.regionserver.HRegionServer; +043import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +044import org.apache.hadoop.hbase.TableName; +045import org.apache.hadoop.hbase.regionserver.Region; +046import org.apache.hadoop.hbase.regionserver.RegionScanner; +047import org.apache.hadoop.hbase.HBaseTestingUtility; +048import org.apache.hadoop.hbase.HColumnDescriptor; +049import org.apache.hadoop.hbase.HConstants; +050import org.apache.hadoop.hbase.HRegionLocation; +051import org.apache.hadoop.hbase.HTableDescriptor; +052import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; +053import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +054import org.apache.hadoop.hbase.ipc.ServerRpcController; +055import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; +056import org.apache.hadoop.hbase.regionserver.HRegion; +057import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +058import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; +059import org.apache.hadoop.hbase.testclassification.ClientTests; +060import org.apache.hadoop.hbase.testclassification.LargeTests; +061import org.apache.hadoop.hbase.util.Bytes; +062import org.apache.hadoop.hbase.util.Pair; +063import org.junit.After; +064import org.junit.AfterClass; +065import static org.junit.Assert.assertEquals; +066import static org.junit.Assert.assertFalse; +067import static org.junit.Assert.assertNull; +068import static org.junit.Assert.assertTrue; +069import static org.junit.Assert.fail; +070import org.junit.Before; +071import org.junit.BeforeClass; +072import org.junit.Rule; +073import org.junit.Test; +074import org.junit.experimental.categories.Category; +075import org.junit.rules.TestName; +076 +077@Category({LargeTests.class, ClientTests.class}) +078public class TestFromClientSide3 { +079 private static final Log LOG = LogFactory.getLog(TestFromClientSide3.class); +080 private final static HBaseTestingUtility TEST_UTIL +081 = new HBaseTestingUtility(); +082 private static byte[] FAMILY = Bytes.toBytes("testFamily"); +083 private static Random random = new Random(); +084 private static int SLAVES = 3; +085 private static byte [] ROW = Bytes.toBytes("testRow"); +086 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow"); +087 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); +088 private static byte [] VALUE = Bytes.toBytes("testValue"); +089 private final static byte[] COL_QUAL = Bytes.toBytes("f1"); +090 private final static byte[] VAL_BYTES = Bytes.toBytes("v1"); +091 private final static byte[] ROW_BYTES = Bytes.toBytes("r1"); +092 +093 @Rule +094 public TestName name = new TestName(); +095 +096 /** +097 * @throws java.lang.Exception +098 */ +099 @BeforeClass +100 public static void setUpBeforeClass() throws Exception { +101 TEST_UTIL.startMiniCluster(SLAVES); +102 } +103 +104 /** +105 * @throws java.lang.Exception +106 */ +107 @AfterClass +108 public static void tearDownAfterClass() throws Exception { +109 TEST_UTIL.shutdownMiniCluster(); +110 } +111 +112 /** +113 * @throws java.lang.Exception +114 */ +115 @Before +116 public void setUp() throws Exception { +117 // Nothing to do. +118 } +119 +120 /** +121 * @throws java.lang.Exception +122 */ +123 @After +124 public void tearDown() throws Exception { +125 for (HTableDescriptor htd: TEST_UTIL.getAdmin().listTables()) { +126 LOG.info("Tear down, remove table=" + htd.getTableName()); +127 TEST_UTIL.deleteTable(htd.getTableName()); 128 } -129 -130 private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) -131 throws Exception { -132 Put put = new Put(row); -133 for (int i = 0; i < nPuts; i++) { -134 byte[] qualifier = Bytes.toBytes(random.nextInt()); -135 byte[] value = Bytes.toBytes(random.nextInt()); -136 put.addColumn(family, qualifier, value); -137 } -138 table.put(put); -139 } -140 -141 private void performMultiplePutAndFlush(HBaseAdmin admin, Table table, -142 byte[] row, byte[] family, int nFlushes, int nPuts) -143 throws Exception { -144 -145 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(table.getName())) { -146 // connection needed for poll-wait -147 HRegionLocation loc = locator.getRegionLocation(row, true); -148 AdminProtos.AdminService.BlockingInterface server = -149 ((ClusterConnection) admin.getConnection()).getAdmin(loc.getServerName()); -150 byte[] regName = loc.getRegionInfo().getRegionName(); -151 -152 for (int i = 0; i < nFlushes; i++) { -153 randomCFPuts(table, row, family, nPuts); -154 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY); -155 int sfCount = sf.size(); -156 -157 admin.flush(table.getName()); -158 } -159 } -160 } -161 -162 // override the config settings at the CF level and ensure priority -163 @Test(timeout = 60000) -164 public void testAdvancedConfigOverride() throws Exception { -165 /* -166 * Overall idea: (1) create 3 store files and issue a compaction. config's -167 * compaction.min == 3, so should work. (2) Increase the compaction.min -168 * toggle in the HTD to 5 and modify table. If we use the HTD value instead -169 * of the default config value, adding 3 files and issuing a compaction -170 * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 -171 * and modify table. The CF schema should override the Table schema and now -172 * cause a minor compaction. -173 */ -174 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); -175 -176 final TableName tableName = TableName.valueOf(name.getMethodName()); -177 Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); -178 Admin admin = TEST_UTIL.getAdmin(); -179 ClusterConnection connection = (ClusterConnection) TEST_UTIL.getConnection(); -180 -181 // Create 3 store files. -182 byte[] row = Bytes.toBytes(random.nextInt()); -183 performMultiplePutAndFlush((HBaseAdmin) admin, hTable, row, FAMILY, 3, 100); -184 -185 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { -186 // Verify we have multiple store files. -187 HRegionLocation loc = locator.getRegionLocation(row, true); -188 byte[] regionName = loc.getRegionInfo().getRegionName(); -189 AdminProtos.AdminService.BlockingInterface server = connection.getAdmin(loc.getServerName()); -190 assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() > 1); -191 -192 // Issue a compaction request -193 admin.compact(tableName); -194 -195 // poll wait for the compactions to happen -196 for (int i = 0; i < 10 * 1000 / 40; ++i) { -197 // The number of store files after compaction should be lesser. -198 loc = locator.getRegionLocation(row, true); -199 if (!loc.getRegionInfo().isOffline()) { -200 regionName = loc.getRegionInfo().getRegionName(); -201 server = connection.getAdmin(loc.getServerName()); -202 if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1) { -203 break; -204 } -205 } -206 Thread.sleep(40); -207 } -208 // verify the compactions took place and that we didn't just time out -209 assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1); -210 -211 // change the compaction.min config option for this table to 5 -212 LOG.info("hbase.hstore.compaction.min should now be 5"); -213 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor()); -214 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); -215 admin.modifyTable(tableName, htd); -216 Pair<Integer, Integer> st; -217 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { -218 LOG.debug(st.getFirst() + " regions left to update"); -219 Thread.sleep(40); -220 } -221 LOG.info("alter status finished"); -222 -223 // Create 3 more store files. -224 performMultiplePutAndFlush((HBaseAdmin) admin, hTable, row, FAMILY, 3, 10); -225 -226 // Issue a compaction request -227 admin.compact(tableName); -228 -229 // This time, the compaction request should not happen -230 Thread.sleep(10 * 1000); -231 loc = locator.getRegionLocation(row, true); -232 regionName = loc.getRegionInfo().getRegionName(); -233 server = connection.getAdmin(loc.getServerName()); -234 int sfCount = ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size(); -235 assertTrue(sfCount > 1); -236 -237 // change an individual CF's config option to 2 & online schema update -238 LOG.info("hbase.hstore.compaction.min should now be 2"); -239 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); -240 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); -241 htd.modifyFamily(hcd); -242 admin.modifyTable(tableName, htd); -243 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { -244 LOG.debug(st.getFirst() + " regions left to update"); -245 Thread.sleep(40); -246 } -247 LOG.info("alter status finished"); -248 -249 // Issue a compaction request -250 admin.compact(tableName); -251 -252 // poll wait for the compactions to happen -253 for (int i = 0; i < 10 * 1000 / 40; ++i) { -254 loc = locator.getRegionLocation(row, true); -255 regionName = loc.getRegionInfo().getRegionName(); -256 try { -257 server = connection.getAdmin(loc.getServerName()); -258 if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount) { -259 break; -260 } -261 } catch (Exception e) { -262 LOG.debug("Waiting for region to come online: " + regionName); -263 } -264 Thread.sleep(40); -265 } -266 -267 // verify the compaction took place and that we didn't just time out -268 assertTrue(ProtobufUtil.getStoreFiles( -269 server, regionName, FAMILY).size() < sfCount); -270 -271 // Finally, ensure that we can remove a custom config value after we made it -272 LOG.info("Removing CF config value"); -273 LOG.info("hbase.hstore.compaction.min should now be 5"); -274 hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); -275 hcd.setValue("hbase.hstore.compaction.min", null); -276 htd.modifyFamily(hcd); -277 admin.modifyTable(tableName, htd); -278 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { -279 LOG.debug(st.getFirst() + " regions left to update"); -280 Thread.sleep(40); -281 } -282 LOG.info("alter status finished"); -283 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue( -284 "hbase.hstore.compaction.min")); -285 } -286 } -287 -288 @Test -289 public void testHTableBatchWithEmptyPut ()throws Exception { -290 Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), -291 new byte[][] { FAMILY }); -292 try { -293 List actions = (List) new ArrayList(); -294 Object[] results = new Object[2]; -295 // create an empty Put -296 Put put1 = new Put(ROW); -297 actions.add(put1); -298 -299 Put put2 = new Put(ANOTHERROW); -300 put2.addColumn(FAMILY, QUALIFIER, VALUE); -301 actions.add(put2); -302 -303 table.batch(actions, results); -304 fail("Empty Put should have failed the batch call"); -305 } catch (IllegalArgumentException iae) { -306 -307 } finally { -308 table.close(); -309 } -310 } +129 } +130 +131 private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) +132 throws Exception { +133 Put put = new Put(row); +134 for (int i = 0; i < nPuts; i++) { +135 byte[] qualifier = Bytes.toBytes(random.nextInt()); +136 byte[] value = Bytes.toBytes(random.nextInt()); +137 put.addColumn(family, qualifier, value); +138 } +139 table.put(put); +140 } +141 +142 private void performMultiplePutAndFlush(HBaseAdmin admin, Table table, +143 byte[] row, byte[] family, int nFlushes, int nPuts) +144 throws Exception { +145 +146 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(table.getName())) { +147 // connection needed for poll-wait +148 HRegionLocation loc = locator.getRegionLocation(row, true); +149 AdminProtos.AdminService.BlockingInterface server = +150 ((ClusterConnection) admin.getConnection()).getAdmin(loc.getServerName()); +151 byte[] regName = loc.getRegionInfo().getRegionName(); +152 +153 for (int i = 0; i < nFlushes; i++) { +154 randomCFPuts(table, row, family, nPuts); +155 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY); +156 int sfCount = sf.size(); +157 +158 admin.flush(table.getName()); +159 } +160 } +161 } +162 +163 private static List<Cell> toList(ResultScanner scanner) { +164 try { +165 List<Cell> cells = new ArrayList<>(); +166 for (Result r : scanner) { +167 cells.addAll(r.listCells()); +168 } +169 return cells; +170 } finally { +171 scanner.close(); +172 } +173 } +174 +175 @Test +176 public void testScanAfterDeletingSpecifiedRow() throws IOException { +177 TableName tableName = TableName.valueOf(name.getMethodName()); +178 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) +179 .addColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)) +180 .build(); +181 TEST_UTIL.getAdmin().createTable(desc); +182 byte[] row = Bytes.toBytes("SpecifiedRow"); +183 byte[] value0 = Bytes.toBytes("value_0"); +184 byte[] value1 = Bytes.toBytes("value_1"); +185 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { +186 Put put = new Put(row); +187 put.addColumn(FAMILY, QUALIFIER, VALUE); +188 t.put(put); +189 Delete d = new Delete(row); +190 t.delete(d); +191 put = new Put(row); +192 put.addColumn(FAMILY, null, value0); +193 t.put(put); +194 put = new Put(row); +195 put.addColumn(FAMILY, null, value1); +196 t.put(put); +197 List<Cell> cells = toList(t.getScanner(new Scan())); +198 assertEquals(1, cells.size()); +199 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); +200 +201 cells = toList(t.getScanner(new Scan().addFamily(FAMILY))); +202 assertEquals(1, cells.size()); +203 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); +204 +205 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); +206 assertEquals(0, cells.size()); +207 +208 TEST_UTIL.getAdmin().flush(tableName); +209 cells = toList(t.getScanner(new Scan())); +210 assertEquals(1, cells.size()); +211 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); +212 +213 cells = toList(t.getScanner(new Scan().addFamily(FAMILY))); +214 assertEquals(1, cells.size()); +215 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0)))); +216 +217 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER))); +218 assertEquals(0, cells.size()); +219 } +220 } +221 +222 @Test +223 public void testScanAfterDeletingSpecifiedRowV2() throws IOException { +224 TableName tableName = TableName.valueOf(name.getMethodName()); +225 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) +226 .addColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)) +227 .build(); +228 TEST_UTIL.getAdmin().createTable(desc); +229 byte[] row = Bytes.toBytes("SpecifiedRow"); +230 byte[] qual0 = Bytes.toBytes("qual0"); +231 byte[] qual1 = Bytes.toBytes("qual1"); +232 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { +233 Delete d = new Delete(row); +234 t.delete(d); +235 +236 Put put = new Put(row); +237 put.addColumn(FAMILY, null, VALUE); +238 t.put(put); +239 +240 put = new Put(row); +241 put.addColumn(FAMILY, qual1, qual1); +242 t.put(put); +243 +244 put = new Put(row); +245 put.addColumn(FAMILY, qual0, qual0); +246 t.put(put); +247 +248 Result r = t.get(new Get(row)); +249 assertEquals(3, r.size()); +250 assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); +251 assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); +252 assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); +253 +254 TEST_UTIL.getAdmin().flush(tableName); +255 r = t.get(new Get(row)); +256 assertEquals(3, r.size()); +257 assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); +258 assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1]))); +259 assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2]))); +260 } +261 } +262 +263 // override the config settings at the CF level and ensure priority +264 @Test(timeout = 60000) +265 public void testAdvancedConfigOverride() throws Exception { +266 /* +267 * Overall idea: (1) create 3 store files and issue a compaction. config's +268 * compaction.min == 3, so should work. (2) Increase the compaction.min +269 * toggle in the HTD to 5 and modify table. If we use the HTD value instead +270 * of the default config value, adding 3 files and issuing a compaction +271 * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 +272 * and modify table. The CF schema should override the Table schema and now +273 * cause a minor compaction. +274 */ +275 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3); +276 +277 final TableName tableName = TableName.valueOf(name.getMethodName()); +278 Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); +279 Admin admin = TEST_UTIL.getAdmin(); +280 ClusterConnection connection = (ClusterConnection) TEST_UTIL.getConnection(); +281 +282 // Create 3 store files. +283 byte[] row = Bytes.toBytes(random.nextInt()); +284 performMultiplePutAndFlush((HBaseAdmin) admin, hTable, row, FAMILY, 3, 100); +285 +286 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { +287 // Verify we have multiple store files. +288 HRegionLocation loc = locator.getRegionLocation(row, true); +289 byte[] regionName = loc.getRegionInfo().getRegionName(); +290 AdminProtos.AdminService.BlockingInterface server = connection.getAdmin(loc.getServerName()); +291 assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() > 1); +292 +293 // Issue a compaction request +294 admin.compact(tableName); +295 +296 // poll wait for the compactions to happen +297 for (int i = 0; i < 10 * 1000 / 40; ++i) { +298 // The number of store files after compaction should be lesser. +299 loc = locator.getRegionLocation(row, true); +300 if (!loc.getRegionInfo().isOffline()) { +301 regionName = loc.getRegionInfo().getRegionName(); +302 server = connection.getAdmin(loc.getServerName()); +303 if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1) { +304 break; +305 } +306 } +307 Thread.sleep(40); +308 } +309 // verify the compactions took place and that we didn't just time out +310 assertTrue(ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() <= 1); 311 -312 @Test -313 public void testBatchWithRowMutation() throws Exception { -314 LOG.info("Starting testBatchWithRowMutation"); -315 final TableName TABLENAME = TableName.valueOf("testBatchWithRowMutation"); -316 try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) { -317 byte [][] QUALIFIERS = new byte [][] { -318 Bytes.toBytes("a"), Bytes.toBytes("b") -319 }; -320 RowMutations arm = new RowMutations(ROW); -321 Put p = new Put(ROW); -322 p.addColumn(FAMILY, QUALIFIERS[0], VALUE); -323 arm.add(p); -324 Object[] batchResult = new Object[1]; -325 t.batch(Arrays.asList(arm), batchResult); +312 // change the compaction.min config option for this table to 5 +313 LOG.info("hbase.hstore.compaction.min should now be 5"); +314 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor()); +315 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5)); +316 admin.modifyTable(tableName, htd); +317 Pair<Integer, Integer> st; +318 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { +319 LOG.debug(st.getFirst() + " regions left to update"); +320 Thread.sleep(40); +321 } +322 LOG.info("alter status finished"); +323 +324 // Create 3 more store files. +325 performMultiplePutAndFlush((HBaseAdmin) admin, hTable, row, FAMILY, 3, 10); 326 -327 Get g = new Get(ROW); -328 Result r = t.get(g); -329 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); -330 -331 arm = new RowMutations(ROW); -332 p = new Put(ROW); -333 p.addColumn(FAMILY, QUALIFIERS[1], VALUE); -334 arm.add(p); -335 Delete d = new Delete(ROW); -336 d.addColumns(FAMILY, QUALIFIERS[0]); -337 arm.add(d); -338 t.batch(Arrays.asList(arm), batchResult); -339 r = t.get(g); -340 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); -341 assertNull(r.getValue(FAMILY, QUALIFIERS[0])); -342 -343 // Test that we get the correct remote exception for RowMutations from batch() -344 try { -345 arm = new RowMutations(ROW); -346 p = new Put(ROW); -347 p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE); -348 arm.add(p); -349 t.batch(Arrays.asList(arm), batchResult); -350 fail("Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException"); -351 } catch(RetriesExhaustedWithDetailsException e) { -352 String msg = e.getMessage(); -353 assertTrue(msg.contains("NoSuchColumnFamilyException")); -354 } -355 } -356 } -357 -358 @Test -359 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception { -360 // Test with a single region table. -361 Table table = TEST_UTIL.createTable( -362 TableName.valueOf(name.getMethodName()), -363 new byte[][] { FAMILY }); -364 -365 Put put = new Put(ROW); -366 put.addColumn(FAMILY, QUALIFIER, VALUE); +327 // Issue a compaction request +328 admin.compact(tableName); +329 +330 // This time, the compaction request should not happen +331 Thread.sleep(10 * 1000); +332 loc = locator.getRegionLocation(row, true); +333 regionName = loc.getRegionInfo().getRegionName(); +334 server = connection.getAdmin(loc.getServerName()); +335 int sfCount = ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size(); +336 assertTrue(sfCount > 1); +337 +338 // change an individual CF's config option to 2 & online schema update +339 LOG.info("hbase.hstore.compaction.min should now be 2"); +340 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); +341 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); +342 htd.modifyFamily(hcd); +343 admin.modifyTable(tableName, htd); +344 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { +345 LOG.debug(st.getFirst() + " regions left to update"); +346 Thread.sleep(40); +347 } +348 LOG.info("alter status finished"); +349 +350 // Issue a compaction request +351 admin.compact(tableName); +352 +353 // poll wait for the compactions to happen +354 for (int i = 0; i < 10 * 1000 / 40; ++i) { +355 loc = locator.getRegionLocation(row, true); +356 regionName = loc.getRegionInfo().getRegionName(); +357 try { +358 server = connection.getAdmin(loc.getServerName()); +359 if (ProtobufUtil.getStoreFiles(server, regionName, FAMILY).size() < sfCount) { +360 break; +361 } +362 } catch (Exception e) { +363 LOG.debug("Waiting for region to come online: " + regionName); +364 } +365 Thread.sleep(40); +366 } 367 -368 Get get = new Get(ROW); -369 -370 boolean exist = table.exists(get); -371 assertEquals(exist, false); -372 -373 table.put(put); -374 -375 exist = table.exists(get); -376 assertEquals(exist, true); -377 } -378 -379 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception { -380 Table table = TEST_UTIL.createTable(TableName.valueOf( -381 name.getMethodName()), new byte[][] { FAMILY }); -382 -383 Put put = new Put(ROW); -384 put.addColumn(FAMILY, QUALIFIER, VALUE); -385 table.put(put); -386 -387 List<Get> gets = new ArrayList<>(); -388 gets.add(new Get(ROW)); -389 gets.add(null); -390 gets.add(new Get(ANOTHERROW)); -391 -392 boolean[] results = table.existsAll(gets); -393 assertEquals(results[0], true); -394 assertEquals(results[1], false); -395 assertEquals(results[2], false); -396 } -397 -398 @Test -399 public void testHTableExistsBeforeGet() throws Exception { -400 Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), -401 new byte[][] { FAMILY }); -402 try { -403 Put put = new Put(ROW); -404 put.addColumn(FAMILY, QUALIFIER, VALUE); -405 table.put(put); -406 -407 Get get = new Get(ROW); -408 -409 boolean exist = table.exists(get); -410 assertEquals(true, exist); -411 -412 Result result = table.get(get); -413 assertEquals(false, result.isEmpty()); -414 assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER))); -415 } finally { -416 table.close(); -417 } -418 } -419 -420 @Test -421 public void testHTableExistsAllBeforeGet() throws Exception { -422 final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2")); -423 Table table = TEST_UTIL.createTable( -424 TableName.valueOf(name.getMethodName()), new byte[][] { FAMILY }); -425 try { -426 Put put = new Put(ROW); -427 put.addColumn(FAMILY, QUALIFIER, VALUE); -428 table.put(put); -429 put = new Put(ROW2); -430 put.addColumn(FAMILY, QUALIFIER, VALUE); -431 table.put(put); -432 -433 Get get = new Get(ROW); -434 Get get2 = new Get(ROW2); -435 ArrayList<Get> getList = new ArrayList(2); -436 getList.add(get); -437 getList.add(get2); -438 -439 boolean[] exists = table.existsAll(getList); -440 assertEquals(true, exists[0]); -441 assertEquals(true, exists[1]); -442 -443 Result[] result = table.get(getList); -444 assertEquals(false, result[0].isEmpty()); -445 assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER))); -446 assertEquals(false, result[1].isEmpty()); -447 assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER))); -448 } finally { -449 table.close(); -450 } -451 } -452 -453 @Test -454 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception { -455 Table table = TEST_UTIL.createTable( -456 TableName.valueOf(name.getMethodName()), new byte[][] { FAMILY }, -457 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255); -458 Put put = new Put(ROW); -459 put.addColumn(FAMILY, QUALIFIER, VALUE); -460 -461 Get get = new Get(ROW); -462 -463 boolean exist = table.exists(get); -464 assertEquals(exist, false); +368 // verify the compaction took place and that we didn't just time out +369 assertTrue(ProtobufUtil.getStoreFiles( +370 server, regionName, FAMILY).size() < sfCount); +371 +372 // Finally, ensure that we can remove a custom config value after we made it +373 LOG.info("Removing CF config value"); +374 LOG.info("hbase.hstore.compaction.min should now be 5"); +375 hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); +376 hcd.setValue("hbase.hstore.compaction.min", null); +377 htd.modifyFamily(hcd); +378 admin.modifyTable(tableName, htd); +379 while (null != (st = admin.getAlterStatus(tableName)) && st.getFirst() > 0) { +380 LOG.debug(st.getFirst() + " regions left to update"); +381 Thread.sleep(40); +382 } +383 LOG.info("alter status finished"); +384 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue( +385 "hbase.hstore.compaction.min")); +386 } +387 } +388 +389 @Test +390 public void testHTableBatchWithEmptyPut ()throws Exception { +391 Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), +392 new byte[][] { FAMILY }); +393 try { +394 List actions = (List) new ArrayList(); +395 Object[] results = new Object[2]; +396 // create an empty Put +397 Put put1 = new Put(ROW); +398 actions.add(put1); +399 +400 Put put2 = new Put(ANOTHERROW); +401 put2.addColumn(FAMILY, QUALIFIER, VALUE); +402 actions.add(put2); +403 +404 table.batch(actions, results); +405 fail("Empty Put should have failed the batch call"); +406 } catch (IllegalArgumentException iae) { +407 +408 } finally { +409 table.close(); +410 } +411 } +412 +413 @Test +414 public void testBatchWithRowMutation() throws Exception { +415 LOG.info("Starting testBatchWithRowMutation"); +416 final TableName TABLENAME = TableName.valueOf("testBatchWithRowMutation"); +417 try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) { +418 byte [][] QUALIFIERS = new byte [][] { +419 Bytes.toBytes("a"), Bytes.toBytes("b") +420 }; +421 RowMutations arm = new RowMutations(ROW); +422 Put p = new Put(ROW); +423 p.addColumn(FAMILY, QUALIFIERS[0], VALUE); +424 arm.add(p); +425 Object[] batchResult = new Object[1]; +426 t.batch(Arrays.asList(arm), batchResult); +427 +428 Get g = new Get(ROW); +429 Result r = t.get(g); +430 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); +431 +432 arm = new RowMutations(ROW); +433 p = new Put(ROW); +434 p.addColumn(FAMILY, QUALIFIERS[1], VALUE); +435 arm.add(p); +436 Delete d = new Delete(ROW); +437 d.addColumns(FAMILY, QUALIFIERS[0]); +438 arm.add(d); +439 t.batch(Arrays.asList(arm), batchResult); +440 r = t.get(g); +441 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); +442 assertNull(r.getValue(FAMILY, QUALIFIERS[0])); +443 +444 // Test that we get the correct remote exception for RowMutations from batch() +445 try { +446 arm = new RowMutations(ROW); +447 p = new Put(ROW); +448 p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE); +449 arm.add(p); +450 t.batch(Arrays.asList(arm), batchResult); +451 fail("Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException"); +452 } catch(RetriesExhaustedWithDetailsException e) { +453 String msg = e.getMessage(); +454 assertTrue(msg.contains("NoSuchColumnFamilyException")); +455 } +456 } +457 } +458 +459 @Test +460 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception { +461 // Test with a single region table. +462 Table table = TEST_UTIL.createTable( +463 TableName.valueOf(name.getMethodName()), +464 new byte[][] { FAMILY }); 465 -466 table.put(put); -467 -468 exist = table.exists(get); -469 assertEquals(exist, true); -470 } -471 -472 @Test -473 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception { -474 Table table = TEST_UTIL.createTable( -475 TableName.valueOf(name.getMethodName()), -476 new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255); -477 Put put = new Put(ROW); -478 put.addColumn(FAMILY, QUALIFIER, VALUE); -479 table.put (put); -480 -481 List<Get> gets = new ArrayList<>(); -482 gets.add(new Get(ANOTHERROW)); -483 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 }))); -484 gets.add(new Get(ROW)); -485 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 }))); -486 -487 LOG.info("Calling exists"); -488 boolean[] results = table.existsAll(gets); -489 assertEquals(results[0], false); -490 assertEquals(results[1], false); -491 assertEquals(results[2], true); -492 assertEquals(results[3], false); -493 -494 // Test with the first region. -495 put = new Put(new byte[] { 0x00 }); -496 put.addColumn(FAMILY, QUALIFIER, VALUE); -497 table.put(put); +466 Put put = new Put(ROW); +467 put.addColumn(FAMILY, QUALIFIER, VALUE); +468 +469 Get get = new Get(ROW); +470 +471 boolean exist = table.exists(get); +472 assertEquals(exist, false); +473 +474 table.put(put);