Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 60A631897E for ; Fri, 18 Dec 2015 16:42:37 +0000 (UTC) Received: (qmail 70752 invoked by uid 500); 18 Dec 2015 16:42:36 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 70429 invoked by uid 500); 18 Dec 2015 16:42:35 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 69673 invoked by uid 99); 18 Dec 2015 16:42:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 18 Dec 2015 16:42:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 11164E698C; Fri, 18 Dec 2015 16:42:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Fri, 18 Dec 2015 16:42:42 -0000 Message-Id: <6228d16a2dc647e9a673781cc6bcb0b6@git.apache.org> In-Reply-To: <2b06700c24844cf38cd4158783c0e9ba@git.apache.org> References: <2b06700c24844cf38cd4158783c0e9ba@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [09/48] hbase-site git commit: Published site at 4bfeccb87a94cfe232ea8fc9a6f40ff5b8d3b1c5. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/d917c66a/xref-test/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.html ---------------------------------------------------------------------- diff --git a/xref-test/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.html b/xref-test/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.html index ee91a78..8fe7c02 100644 --- a/xref-test/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.html +++ b/xref-test/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.html @@ -28,937 +28,636 @@ 18 */ 19 package org.apache.hadoop.hbase.util; 20 -21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; -22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors; -23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; -24 import static org.junit.Assert.assertEquals; -25 import static org.junit.Assert.assertFalse; -26 import static org.junit.Assert.assertNotEquals; -27 import static org.junit.Assert.assertNotNull; -28 import static org.junit.Assert.assertTrue; -29 import static org.junit.Assert.fail; -30 -31 import java.io.IOException; -32 import java.util.ArrayList; -33 import java.util.Collection; -34 import java.util.HashMap; -35 import java.util.List; -36 import java.util.Map; -37 import java.util.UUID; -38 import java.util.concurrent.CountDownLatch; -39 import java.util.concurrent.ExecutorService; -40 import java.util.concurrent.ScheduledThreadPoolExecutor; -41 import java.util.concurrent.atomic.AtomicBoolean; -42 -43 import org.apache.commons.logging.Log; -44 import org.apache.commons.logging.LogFactory; -45 import org.apache.hadoop.conf.Configuration; -46 import org.apache.hadoop.fs.FileStatus; -47 import org.apache.hadoop.fs.FileSystem; -48 import org.apache.hadoop.fs.Path; -49 import org.apache.hadoop.hbase.ClusterStatus; -50 import org.apache.hadoop.hbase.HBaseTestingUtility; -51 import org.apache.hadoop.hbase.HColumnDescriptor; -52 import org.apache.hadoop.hbase.HConstants; -53 import org.apache.hadoop.hbase.HRegionInfo; -54 import org.apache.hadoop.hbase.HRegionLocation; -55 import org.apache.hadoop.hbase.HTableDescriptor; -56 import org.apache.hadoop.hbase.MiniHBaseCluster; -57 import org.apache.hadoop.hbase.ServerName; -58 import org.apache.hadoop.hbase.TableName; -59 import org.apache.hadoop.hbase.MetaTableAccessor; -60 import org.apache.hadoop.hbase.client.Admin; -61 import org.apache.hadoop.hbase.client.ClusterConnection; -62 import org.apache.hadoop.hbase.client.Connection; -63 import org.apache.hadoop.hbase.client.ConnectionFactory; -64 import org.apache.hadoop.hbase.client.Delete; -65 import org.apache.hadoop.hbase.client.Get; -66 import org.apache.hadoop.hbase.client.HBaseAdmin; -67 import org.apache.hadoop.hbase.client.HConnection; -68 import org.apache.hadoop.hbase.client.Put; -69 import org.apache.hadoop.hbase.client.RegionLocator; -70 import org.apache.hadoop.hbase.client.ResultScanner; -71 import org.apache.hadoop.hbase.client.Scan; -72 import org.apache.hadoop.hbase.client.Table; -73 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; -74 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -75 import org.apache.hadoop.hbase.coprocessor.ObserverContext; -76 import org.apache.hadoop.hbase.master.AssignmentManager; -77 import org.apache.hadoop.hbase.master.RegionState; -78 import org.apache.hadoop.hbase.master.RegionStates; -79 import org.apache.hadoop.hbase.mob.MobFileName; -80 import org.apache.hadoop.hbase.mob.MobUtils; -81 import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -82 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; -83 import org.apache.hadoop.hbase.regionserver.HRegion; -84 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -85 import org.apache.hadoop.hbase.regionserver.HRegionServer; -86 import org.apache.hadoop.hbase.regionserver.SplitTransactionFactory; -87 import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl; -88 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; -89 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; -90 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; -91 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo; -92 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; -93 import org.apache.zookeeper.KeeperException; -94 import org.junit.Assert; -95 import org.junit.Before; -96 import org.junit.Ignore; -97 import org.junit.Test; -98 import org.junit.rules.TestName; -99 -100 /** -101 * This is the base class for HBaseFsck's ability to detect reasons for inconsistent tables. -102 * -103 * Actual tests are in : -104 * TestHBaseFsckTwoRS -105 * TestHBaseFsckOneRS -106 * TestHBaseFsckMOB -107 * TestHBaseFsckReplicas -108 */ -109 public class BaseTestHBaseFsck { -110 static final int POOL_SIZE = 7; -111 protected static final Log LOG = LogFactory.getLog(BaseTestHBaseFsck.class); -112 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); -113 protected final static Configuration conf = TEST_UTIL.getConfiguration(); -114 protected final static String FAM_STR = "fam"; -115 protected final static byte[] FAM = Bytes.toBytes(FAM_STR); -116 protected final static int REGION_ONLINE_TIMEOUT = 800; -117 protected static RegionStates regionStates; -118 protected static ExecutorService tableExecutorService; -119 protected static ScheduledThreadPoolExecutor hbfsckExecutorService; -120 protected static ClusterConnection connection; -121 protected static Admin admin; -122 -123 // for the instance, reset every test run -124 protected Table tbl; -125 protected final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"), -126 Bytes.toBytes("B"), Bytes.toBytes("C") }; -127 // one row per region. -128 protected final static byte[][] ROWKEYS= new byte[][] { -129 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), -130 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; -131 -132 -133 /** -134 * Create a new region in META. -135 */ -136 protected HRegionInfo createRegion(final HTableDescriptor -137 htd, byte[] startKey, byte[] endKey) -138 throws IOException { -139 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); -140 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); -141 MetaTableAccessor.addRegionToMeta(meta, hri); -142 meta.close(); -143 return hri; -144 } -145 -146 /** -147 * Debugging method to dump the contents of meta. -148 */ -149 protected void dumpMeta(TableName tableName) throws IOException { -150 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName); -151 for (byte[] row : metaRows) { -152 LOG.info(Bytes.toString(row)); -153 } -154 } -155 -156 /** -157 * This method is used to undeploy a region -- close it and attempt to -158 * remove its state from the Master. -159 */ -160 protected void undeployRegion(Connection conn, ServerName sn, -161 HRegionInfo hri) throws IOException, InterruptedException { -162 try { -163 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri); -164 if (!hri.isMetaTable()) { -165 admin.offline(hri.getRegionName()); -166 } -167 } catch (IOException ioe) { -168 LOG.warn("Got exception when attempting to offline region " -169 + Bytes.toString(hri.getRegionName()), ioe); -170 } -171 } -172 /** -173 * Delete a region from assignments, meta, or completely from hdfs. -174 * @param unassign if true unassign region if assigned -175 * @param metaRow if true remove region's row from META -176 * @param hdfs if true remove region's dir in HDFS -177 */ -178 protected void deleteRegion(Configuration conf, final HTableDescriptor htd, -179 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, -180 boolean hdfs) throws IOException, InterruptedException { -181 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, -182 HRegionInfo.DEFAULT_REPLICA_ID); -183 } -184 -185 /** -186 * Delete a region from assignments, meta, or completely from hdfs. -187 * @param unassign if true unassign region if assigned -188 * @param metaRow if true remove region's row from META -189 * @param hdfs if true remove region's dir in HDFS -190 * @param regionInfoOnly if true remove a region dir's .regioninfo file -191 * @param replicaId replica id -192 */ -193 protected void deleteRegion(Configuration conf, final HTableDescriptor htd, -194 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, -195 boolean hdfs, boolean regionInfoOnly, int replicaId) -196 throws IOException, InterruptedException { -197 LOG.info("** Before delete:"); -198 dumpMeta(htd.getTableName()); -199 -200 List<HRegionLocation> locations; -201 try(RegionLocator rl = connection.getRegionLocator(tbl.getName())) { -202 locations = rl.getAllRegionLocations(); -203 } -204 -205 for (HRegionLocation location : locations) { -206 HRegionInfo hri = location.getRegionInfo(); -207 ServerName hsa = location.getServerName(); -208 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 -209 && Bytes.compareTo(hri.getEndKey(), endKey) == 0 -210 && hri.getReplicaId() == replicaId) { -211 -212 LOG.info("RegionName: " +hri.getRegionNameAsString()); -213 byte[] deleteRow = hri.getRegionName(); -214 -215 if (unassign) { -216 LOG.info("Undeploying region " + hri + " from server " + hsa); -217 undeployRegion(connection, hsa, hri); +21 import static org.junit.Assert.assertEquals; +22 import static org.junit.Assert.fail; +23 +24 import java.io.IOException; +25 import java.util.ArrayList; +26 import java.util.Collection; +27 import java.util.HashMap; +28 import java.util.List; +29 import java.util.Map; +30 import java.util.UUID; +31 import java.util.concurrent.CountDownLatch; +32 import java.util.concurrent.ExecutorService; +33 import java.util.concurrent.ScheduledThreadPoolExecutor; +34 +35 import org.apache.commons.logging.Log; +36 import org.apache.commons.logging.LogFactory; +37 import org.apache.hadoop.conf.Configuration; +38 import org.apache.hadoop.fs.FileStatus; +39 import org.apache.hadoop.fs.FileSystem; +40 import org.apache.hadoop.fs.Path; +41 import org.apache.hadoop.hbase.ClusterStatus; +42 import org.apache.hadoop.hbase.HBaseTestingUtility; +43 import org.apache.hadoop.hbase.HColumnDescriptor; +44 import org.apache.hadoop.hbase.HConstants; +45 import org.apache.hadoop.hbase.HRegionInfo; +46 import org.apache.hadoop.hbase.HRegionLocation; +47 import org.apache.hadoop.hbase.HTableDescriptor; +48 import org.apache.hadoop.hbase.ServerName; +49 import org.apache.hadoop.hbase.TableName; +50 import org.apache.hadoop.hbase.MetaTableAccessor; +51 import org.apache.hadoop.hbase.client.Admin; +52 import org.apache.hadoop.hbase.client.ClusterConnection; +53 import org.apache.hadoop.hbase.client.Connection; +54 import org.apache.hadoop.hbase.client.ConnectionFactory; +55 import org.apache.hadoop.hbase.client.Delete; +56 import org.apache.hadoop.hbase.client.HConnection; +57 import org.apache.hadoop.hbase.client.Put; +58 import org.apache.hadoop.hbase.client.RegionLocator; +59 import org.apache.hadoop.hbase.client.ResultScanner; +60 import org.apache.hadoop.hbase.client.Scan; +61 import org.apache.hadoop.hbase.client.Table; +62 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +63 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +64 import org.apache.hadoop.hbase.coprocessor.ObserverContext; +65 import org.apache.hadoop.hbase.master.RegionStates; +66 import org.apache.hadoop.hbase.mob.MobFileName; +67 import org.apache.hadoop.hbase.mob.MobUtils; +68 import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +69 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; +70 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +71 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; +72 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; +73 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo; +74 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; +75 import org.apache.zookeeper.KeeperException; +76 import org.junit.rules.TestName; +77 +78 /** +79 * This is the base class for HBaseFsck's ability to detect reasons for inconsistent tables. +80 * +81 * Actual tests are in : +82 * TestHBaseFsckTwoRS +83 * TestHBaseFsckOneRS +84 * TestHBaseFsckMOB +85 * TestHBaseFsckReplicas +86 */ +87 public class BaseTestHBaseFsck { +88 static final int POOL_SIZE = 7; +89 protected static final Log LOG = LogFactory.getLog(BaseTestHBaseFsck.class); +90 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); +91 protected final static Configuration conf = TEST_UTIL.getConfiguration(); +92 protected final static String FAM_STR = "fam"; +93 protected final static byte[] FAM = Bytes.toBytes(FAM_STR); +94 protected final static int REGION_ONLINE_TIMEOUT = 800; +95 protected static RegionStates regionStates; +96 protected static ExecutorService tableExecutorService; +97 protected static ScheduledThreadPoolExecutor hbfsckExecutorService; +98 protected static ClusterConnection connection; +99 protected static Admin admin; +100 +101 // for the instance, reset every test run +102 protected Table tbl; +103 protected final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"), +104 Bytes.toBytes("B"), Bytes.toBytes("C") }; +105 // one row per region. +106 protected final static byte[][] ROWKEYS= new byte[][] { +107 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), +108 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; +109 +110 +111 /** +112 * Create a new region in META. +113 */ +114 protected HRegionInfo createRegion(final HTableDescriptor +115 htd, byte[] startKey, byte[] endKey) +116 throws IOException { +117 Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); +118 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); +119 MetaTableAccessor.addRegionToMeta(meta, hri); +120 meta.close(); +121 return hri; +122 } +123 +124 /** +125 * Debugging method to dump the contents of meta. +126 */ +127 protected void dumpMeta(TableName tableName) throws IOException { +128 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName); +129 for (byte[] row : metaRows) { +130 LOG.info(Bytes.toString(row)); +131 } +132 } +133 +134 /** +135 * This method is used to undeploy a region -- close it and attempt to +136 * remove its state from the Master. +137 */ +138 protected void undeployRegion(Connection conn, ServerName sn, +139 HRegionInfo hri) throws IOException, InterruptedException { +140 try { +141 HBaseFsckRepair.closeRegionSilentlyAndWait((HConnection) conn, sn, hri); +142 if (!hri.isMetaTable()) { +143 admin.offline(hri.getRegionName()); +144 } +145 } catch (IOException ioe) { +146 LOG.warn("Got exception when attempting to offline region " +147 + Bytes.toString(hri.getRegionName()), ioe); +148 } +149 } +150 /** +151 * Delete a region from assignments, meta, or completely from hdfs. +152 * @param unassign if true unassign region if assigned +153 * @param metaRow if true remove region's row from META +154 * @param hdfs if true remove region's dir in HDFS +155 */ +156 protected void deleteRegion(Configuration conf, final HTableDescriptor htd, +157 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, +158 boolean hdfs) throws IOException, InterruptedException { +159 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false, +160 HRegionInfo.DEFAULT_REPLICA_ID); +161 } +162 +163 /** +164 * Delete a region from assignments, meta, or completely from hdfs. +165 * @param unassign if true unassign region if assigned +166 * @param metaRow if true remove region's row from META +167 * @param hdfs if true remove region's dir in HDFS +168 * @param regionInfoOnly if true remove a region dir's .regioninfo file +169 * @param replicaId replica id +170 */ +171 protected void deleteRegion(Configuration conf, final HTableDescriptor htd, +172 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow, +173 boolean hdfs, boolean regionInfoOnly, int replicaId) +174 throws IOException, InterruptedException { +175 LOG.info("** Before delete:"); +176 dumpMeta(htd.getTableName()); +177 +178 List<HRegionLocation> locations; +179 try(RegionLocator rl = connection.getRegionLocator(tbl.getName())) { +180 locations = rl.getAllRegionLocations(); +181 } +182 +183 for (HRegionLocation location : locations) { +184 HRegionInfo hri = location.getRegionInfo(); +185 ServerName hsa = location.getServerName(); +186 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 +187 && Bytes.compareTo(hri.getEndKey(), endKey) == 0 +188 && hri.getReplicaId() == replicaId) { +189 +190 LOG.info("RegionName: " +hri.getRegionNameAsString()); +191 byte[] deleteRow = hri.getRegionName(); +192 +193 if (unassign) { +194 LOG.info("Undeploying region " + hri + " from server " + hsa); +195 undeployRegion(connection, hsa, hri); +196 } +197 +198 if (regionInfoOnly) { +199 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString()); +200 Path rootDir = FSUtils.getRootDir(conf); +201 FileSystem fs = rootDir.getFileSystem(conf); +202 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), +203 hri.getEncodedName()); +204 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE); +205 fs.delete(hriPath, true); +206 } +207 +208 if (hdfs) { +209 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); +210 Path rootDir = FSUtils.getRootDir(conf); +211 FileSystem fs = rootDir.getFileSystem(conf); +212 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), +213 hri.getEncodedName()); +214 HBaseFsck.debugLsr(conf, p); +215 boolean success = fs.delete(p, true); +216 LOG.info("Deleted " + p + " sucessfully? " + success); +217 HBaseFsck.debugLsr(conf, p); 218 } 219 -220 if (regionInfoOnly) { -221 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString()); -222 Path rootDir = FSUtils.getRootDir(conf); -223 FileSystem fs = rootDir.getFileSystem(conf); -224 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), -225 hri.getEncodedName()); -226 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE); -227 fs.delete(hriPath, true); -228 } +220 if (metaRow) { +221 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) { +222 Delete delete = new Delete(deleteRow); +223 meta.delete(delete); +224 } +225 } +226 } +227 LOG.info(hri.toString() + hsa.toString()); +228 } 229 -230 if (hdfs) { -231 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); -232 Path rootDir = FSUtils.getRootDir(conf); -233 FileSystem fs = rootDir.getFileSystem(conf); -234 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), -235 hri.getEncodedName()); -236 HBaseFsck.debugLsr(conf, p); -237 boolean success = fs.delete(p, true); -238 LOG.info("Deleted " + p + " sucessfully? " + success); -239 HBaseFsck.debugLsr(conf, p); -240 } -241 -242 if (metaRow) { -243 try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) { -244 Delete delete = new Delete(deleteRow); -245 meta.delete(delete); -246 } -247 } -248 } -249 LOG.info(hri.toString() + hsa.toString()); -250 } -251 -252 TEST_UTIL.getMetaTableRows(htd.getTableName()); -253 LOG.info("*** After delete:"); -254 dumpMeta(htd.getTableName()); -255 } -256 -257 /** -258 * Setup a clean table before we start mucking with it. -259 * -260 * It will set tbl which needs to be closed after test -261 * -262 * @throws IOException -263 * @throws InterruptedException -264 * @throws KeeperException -265 */ -266 void setupTable(TableName tablename) throws Exception { -267 setupTableWithRegionReplica(tablename, 1); -268 } -269 -270 /** -271 * Setup a clean table with a certain region_replica count -272 * -273 * It will set tbl which needs to be closed after test +230 TEST_UTIL.getMetaTableRows(htd.getTableName()); +231 LOG.info("*** After delete:"); +232 dumpMeta(htd.getTableName()); +233 } +234 +235 /** +236 * Setup a clean table before we start mucking with it. +237 * +238 * It will set tbl which needs to be closed after test +239 * +240 * @throws IOException +241 * @throws InterruptedException +242 * @throws KeeperException +243 */ +244 void setupTable(TableName tablename) throws Exception { +245 setupTableWithRegionReplica(tablename, 1); +246 } +247 +248 /** +249 * Setup a clean table with a certain region_replica count +250 * +251 * It will set tbl which needs to be closed after test +252 * +253 * @throws Exception +254 */ +255 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception { +256 HTableDescriptor desc = new HTableDescriptor(tablename); +257 desc.setRegionReplication(replicaCount); +258 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); +259 desc.addFamily(hcd); // If a table has no CF's it doesn't get checked +260 createTable(TEST_UTIL, desc, SPLITS); +261 +262 tbl = connection.getTable(tablename, tableExecutorService); +263 List<Put> puts = new ArrayList<Put>(); +264 for (byte[] row : ROWKEYS) { +265 Put p = new Put(row); +266 p.addColumn(FAM, Bytes.toBytes("val"), row); +267 puts.add(p); +268 } +269 tbl.put(puts); +270 } +271 +272 /** +273 * Setup a clean table with a mob-enabled column. 274 * -275 * @throws Exception -276 */ -277 void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception { -278 HTableDescriptor desc = new HTableDescriptor(tablename); -279 desc.setRegionReplication(replicaCount); +275 * @param tablename The name of a table to be created. +276 * @throws Exception +277 */ +278 void setupMobTable(TableName tablename) throws Exception { +279 HTableDescriptor desc = new HTableDescriptor(tablename); 280 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); -281 desc.addFamily(hcd); // If a table has no CF's it doesn't get checked -282 createTable(TEST_UTIL, desc, SPLITS); -283 -284 tbl = connection.getTable(tablename, tableExecutorService); -285 List<Put> puts = new ArrayList<Put>(); -286 for (byte[] row : ROWKEYS) { -287 Put p = new Put(row); -288 p.addColumn(FAM, Bytes.toBytes("val"), row); -289 puts.add(p); -290 } -291 tbl.put(puts); -292 } -293 -294 /** -295 * Setup a clean table with a mob-enabled column. -296 * -297 * @param tablename The name of a table to be created. -298 * @throws Exception -299 */ -300 void setupMobTable(TableName tablename) throws Exception { -301 HTableDescriptor desc = new HTableDescriptor(tablename); -302 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); -303 hcd.setMobEnabled(true); -304 hcd.setMobThreshold(0); -305 desc.addFamily(hcd); // If a table has no CF's it doesn't get checked -306 createTable(TEST_UTIL, desc, SPLITS); -307 -308 tbl = connection.getTable(tablename, tableExecutorService); -309 List<Put> puts = new ArrayList<Put>(); -310 for (byte[] row : ROWKEYS) { -311 Put p = new Put(row); -312 p.addColumn(FAM, Bytes.toBytes("val"), row); -313 puts.add(p); -314 } -315 tbl.put(puts); -316 } -317 -318 /** -319 * Counts the number of row to verify data loss or non-dataloss. -320 */ -321 int countRows() throws IOException { -322 Scan s = new Scan(); -323 ResultScanner rs = tbl.getScanner(s); -324 int i = 0; -325 while(rs.next() !=null) { -326 i++; -327 } -328 return i; -329 } -330 -331 /** -332 * delete table in preparation for next test -333 * -334 * @param tablename -335 * @throws IOException -336 */ -337 void cleanupTable(TableName tablename) throws Exception { -338 if (tbl != null) { -339 tbl.close(); -340 tbl = null; -341 } -342 -343 ((ClusterConnection) connection).clearRegionCache(); -344 deleteTable(TEST_UTIL, tablename); +281 hcd.setMobEnabled(true); +282 hcd.setMobThreshold(0); +283 desc.addFamily(hcd); // If a table has no CF's it doesn't get checked +284 createTable(TEST_UTIL, desc, SPLITS); +285 +286 tbl = connection.getTable(tablename, tableExecutorService); +287 List<Put> puts = new ArrayList<Put>(); +288 for (byte[] row : ROWKEYS) { +289 Put p = new Put(row); +290 p.addColumn(FAM, Bytes.toBytes("val"), row); +291 puts.add(p); +292 } +293 tbl.put(puts); +294 } +295 +296 /** +297 * Counts the number of row to verify data loss or non-dataloss. +298 */ +299 int countRows() throws IOException { +300 Scan s = new Scan(); +301 ResultScanner rs = tbl.getScanner(s); +302 int i = 0; +303 while(rs.next() !=null) { +304 i++; +305 } +306 return i; +307 } +308 +309 /** +310 * delete table in preparation for next test +311 * +312 * @param tablename +313 * @throws IOException +314 */ +315 void cleanupTable(TableName tablename) throws Exception { +316 if (tbl != null) { +317 tbl.close(); +318 tbl = null; +319 } +320 +321 ((ClusterConnection) connection).clearRegionCache(); +322 deleteTable(TEST_UTIL, tablename); +323 } +324 +325 /** +326 * Get region info from local cluster. +327 */ +328 Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException { +329 ClusterStatus status = admin.getClusterStatus(); +330 Collection<ServerName> regionServers = status.getServers(); +331 Map<ServerName, List<String>> mm = +332 new HashMap<ServerName, List<String>>(); +333 for (ServerName hsi : regionServers) { +334 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi); +335 +336 // list all online regions from this region server +337 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server); +338 List<String> regionNames = new ArrayList<String>(); +339 for (HRegionInfo hri : regions) { +340 regionNames.add(hri.getRegionNameAsString()); +341 } +342 mm.put(hsi, regionNames); +343 } +344 return mm; 345 } 346 347 /** -348 * Get region info from local cluster. +348 * Returns the HSI a region info is on. 349 */ -350 Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException { -351 ClusterStatus status = admin.getClusterStatus(); -352 Collection<ServerName> regionServers = status.getServers(); -353 Map<ServerName, List<String>> mm = -354 new HashMap<ServerName, List<String>>(); -355 for (ServerName hsi : regionServers) { -356 AdminProtos.AdminService.BlockingInterface server = ((HConnection) connection).getAdmin(hsi); -357 -358 // list all online regions from this region server -359 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server); -360 List<String> regionNames = new ArrayList<String>(); -361 for (HRegionInfo hri : regions) { -362 regionNames.add(hri.getRegionNameAsString()); -363 } -364 mm.put(hsi, regionNames); -365 } -366 return mm; -367 } -368 -369 /** -370 * Returns the HSI a region info is on. -371 */ -372 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) { -373 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) { -374 if (e.getValue().contains(hri.getRegionNameAsString())) { -375 return e.getKey(); -376 } -377 } -378 return null; -379 } +350 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) { +351 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) { +352 if (e.getValue().contains(hri.getRegionNameAsString())) { +353 return e.getKey(); +354 } +355 } +356 return null; +357 } +358 +359 public void deleteTableDir(TableName table) throws IOException { +360 Path rootDir = FSUtils.getRootDir(conf); +361 FileSystem fs = rootDir.getFileSystem(conf); +362 Path p = FSUtils.getTableDir(rootDir, table); +363 HBaseFsck.debugLsr(conf, p); +364 boolean success = fs.delete(p, true); +