From commits-return-77232-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Sat Aug 25 16:47:59 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id C4257180771 for ; Sat, 25 Aug 2018 16:47:56 +0200 (CEST) Received: (qmail 38495 invoked by uid 500); 25 Aug 2018 14:47:54 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 37982 invoked by uid 99); 25 Aug 2018 14:47:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 25 Aug 2018 14:47:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 08D8DE1133; Sat, 25 Aug 2018 14:47:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 25 Aug 2018 14:48:07 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/37] hbase-site git commit: Published site at 409e742ac3bdbff027b136a87339f4f5511da07d. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c7b180e2/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.HdfsEntry.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.HdfsEntry.html b/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.HdfsEntry.html index cd509b8..a957d31 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.HdfsEntry.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.HdfsEntry.html @@ -152,5137 +152,5182 @@ 144import org.apache.hadoop.util.ReflectionUtils; 145import org.apache.hadoop.util.Tool; 146import org.apache.hadoop.util.ToolRunner; -147import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -148import org.apache.yetus.audience.InterfaceAudience; -149import org.apache.yetus.audience.InterfaceStability; -150import org.apache.zookeeper.KeeperException; -151import org.slf4j.Logger; -152import org.slf4j.LoggerFactory; -153 -154import org.apache.hbase.thirdparty.com.google.common.base.Joiner; -155import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; -156import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; -157import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -158import org.apache.hbase.thirdparty.com.google.common.collect.Multimap; -159import org.apache.hbase.thirdparty.com.google.common.collect.Ordering; -160import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap; -161 -162import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -163import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface; -164 -165/** -166 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and -167 * table integrity problems in a corrupted HBase. -168 * <p> -169 * Region consistency checks verify that hbase:meta, region deployment on region -170 * servers and the state of data in HDFS (.regioninfo files) all are in -171 * accordance. +147import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +148import org.apache.hbase.thirdparty.com.google.common.collect.Sets; +149import org.apache.yetus.audience.InterfaceAudience; +150import org.apache.yetus.audience.InterfaceStability; +151import org.apache.zookeeper.KeeperException; +152import org.slf4j.Logger; +153import org.slf4j.LoggerFactory; +154 +155import org.apache.hbase.thirdparty.com.google.common.base.Joiner; +156import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +157import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; +158import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +159import org.apache.hbase.thirdparty.com.google.common.collect.Multimap; +160import org.apache.hbase.thirdparty.com.google.common.collect.Ordering; +161import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap; +162 +163import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +164import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface; +165 +166/** +167 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and +168 * table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not +169 * work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'. +170 * See hbck2 (HBASE-19121) for a hbck tool for hbase2. +171 * 172 * <p> -173 * Table integrity checks verify that all possible row keys resolve to exactly -174 * one region of a table. This means there are no individual degenerate -175 * or backwards regions; no holes between regions; and that there are no -176 * overlapping regions. -177 * <p> -178 * The general repair strategy works in two phases: -179 * <ol> -180 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions) -181 * <li> Repair Region Consistency with hbase:meta and assignments -182 * </ol> -183 * <p> -184 * For table integrity repairs, the tables' region directories are scanned -185 * for .regioninfo files. Each table's integrity is then verified. If there -186 * are any orphan regions (regions with no .regioninfo files) or holes, new -187 * regions are fabricated. Backwards regions are sidelined as well as empty -188 * degenerate (endkey==startkey) regions. If there are any overlapping regions, -189 * a new region is created and all data is merged into the new region. -190 * <p> -191 * Table integrity repairs deal solely with HDFS and could potentially be done -192 * offline -- the hbase region servers or master do not need to be running. -193 * This phase can eventually be used to completely reconstruct the hbase:meta table in -194 * an offline fashion. -195 * <p> -196 * Region consistency requires three conditions -- 1) valid .regioninfo file -197 * present in an HDFS region dir, 2) valid row with .regioninfo data in META, -198 * and 3) a region is deployed only at the regionserver that was assigned to -199 * with proper state in the master. -200 * <p> -201 * Region consistency repairs require hbase to be online so that hbck can -202 * contact the HBase master and region servers. The hbck#connect() method must -203 * first be called successfully. Much of the region consistency information -204 * is transient and less risky to repair. -205 * <p> -206 * If hbck is run from the command line, there are a handful of arguments that -207 * can be used to limit the kinds of repairs hbck will do. See the code in -208 * {@link #printUsageAndExit()} for more details. -209 */ -210@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -211@InterfaceStability.Evolving -212public class HBaseFsck extends Configured implements Closeable { -213 public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute -214 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; -215 private static final int MAX_NUM_THREADS = 50; // #threads to contact regions -216 private static boolean rsSupportsOffline = true; -217 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; -218 private static final int DEFAULT_MAX_MERGE = 5; -219 private static final String TO_BE_LOADED = "to_be_loaded"; -220 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; -221 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; -222 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds -223 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds -224 // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. -225 // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for -226 // AlreadyBeingCreatedException which is implies timeout on this operations up to -227 // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). -228 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds -229 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5; -230 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds -231 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds -232 -233 /********************** -234 * Internal resources -235 **********************/ -236 private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName()); -237 private ClusterMetrics status; -238 private ClusterConnection connection; -239 private Admin admin; -240 private Table meta; -241 // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions -242 protected ExecutorService executor; -243 private long startMillis = EnvironmentEdgeManager.currentTime(); -244 private HFileCorruptionChecker hfcc; -245 private int retcode = 0; -246 private Path HBCK_LOCK_PATH; -247 private FSDataOutputStream hbckOutFd; -248 // This lock is to prevent cleanup of balancer resources twice between -249 // ShutdownHook and the main code. We cleanup only if the connect() is -250 // successful -251 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false); -252 -253 // Unsupported options in HBase 2.0+ -254 private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix", -255 "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans", -256 "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents", -257 "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge"); -258 -259 /*********** -260 * Options -261 ***********/ -262 private static boolean details = false; // do we display the full report -263 private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older -264 private static boolean forceExclusive = false; // only this hbck can modify HBase -265 private boolean fixAssignments = false; // fix assignment errors? -266 private boolean fixMeta = false; // fix meta errors? -267 private boolean checkHdfs = true; // load and check fs consistency? -268 private boolean fixHdfsHoles = false; // fix fs holes? -269 private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) -270 private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) -271 private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) -272 private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs -273 private boolean fixSplitParents = false; // fix lingering split parents -274 private boolean removeParents = false; // remove split parents -275 private boolean fixReferenceFiles = false; // fix lingering reference store file -276 private boolean fixHFileLinks = false; // fix lingering HFileLinks -277 private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows -278 private boolean fixReplication = false; // fix undeleted replication queues for removed peer -279 private boolean cleanReplicationBarrier = false; // clean replication barriers of a table -280 private boolean fixAny = false; // Set to true if any of the fix is required. -281 -282 // limit checking/fixes to listed tables, if empty attempt to check/fix all -283 // hbase:meta are always checked -284 private Set<TableName> tablesIncluded = new HashSet<>(); -285 private TableName cleanReplicationBarrierTable; -286 private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge -287 // maximum number of overlapping regions to sideline -288 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; -289 private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions -290 private Path sidelineDir = null; -291 -292 private boolean rerun = false; // if we tried to fix something, rerun hbck -293 private static boolean summary = false; // if we want to print less output -294 private boolean checkMetaOnly = false; -295 private boolean checkRegionBoundaries = false; -296 private boolean ignorePreCheckPermission = false; // if pre-check permission -297 -298 /********* -299 * State -300 *********/ -301 final private ErrorReporter errors; -302 int fixes = 0; -303 -304 /** -305 * This map contains the state of all hbck items. It maps from encoded region -306 * name to HbckInfo structure. The information contained in HbckInfo is used -307 * to detect and correct consistency (hdfs/meta/deployment) problems. -308 */ -309 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<>(); -310 // Empty regioninfo qualifiers in hbase:meta -311 private Set<Result> emptyRegionInfoQualifiers = new HashSet<>(); +173 * Region consistency checks verify that hbase:meta, region deployment on region +174 * servers and the state of data in HDFS (.regioninfo files) all are in +175 * accordance. +176 * <p> +177 * Table integrity checks verify that all possible row keys resolve to exactly +178 * one region of a table. This means there are no individual degenerate +179 * or backwards regions; no holes between regions; and that there are no +180 * overlapping regions. +181 * <p> +182 * The general repair strategy works in two phases: +183 * <ol> +184 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions) +185 * <li> Repair Region Consistency with hbase:meta and assignments +186 * </ol> +187 * <p> +188 * For table integrity repairs, the tables' region directories are scanned +189 * for .regioninfo files. Each table's integrity is then verified. If there +190 * are any orphan regions (regions with no .regioninfo files) or holes, new +191 * regions are fabricated. Backwards regions are sidelined as well as empty +192 * degenerate (endkey==startkey) regions. If there are any overlapping regions, +193 * a new region is created and all data is merged into the new region. +194 * <p> +195 * Table integrity repairs deal solely with HDFS and could potentially be done +196 * offline -- the hbase region servers or master do not need to be running. +197 * This phase can eventually be used to completely reconstruct the hbase:meta table in +198 * an offline fashion. +199 * <p> +200 * Region consistency requires three conditions -- 1) valid .regioninfo file +201 * present in an HDFS region dir, 2) valid row with .regioninfo data in META, +202 * and 3) a region is deployed only at the regionserver that was assigned to +203 * with proper state in the master. +204 * <p> +205 * Region consistency repairs require hbase to be online so that hbck can +206 * contact the HBase master and region servers. The hbck#connect() method must +207 * first be called successfully. Much of the region consistency information +208 * is transient and less risky to repair. +209 * <p> +210 * If hbck is run from the command line, there are a handful of arguments that +211 * can be used to limit the kinds of repairs hbck will do. See the code in +212 * {@link #printUsageAndExit()} for more details. +213 */ +214@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +215@InterfaceStability.Evolving +216public class HBaseFsck extends Configured implements Closeable { +217 public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute +218 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; +219 private static final int MAX_NUM_THREADS = 50; // #threads to contact regions +220 private static boolean rsSupportsOffline = true; +221 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; +222 private static final int DEFAULT_MAX_MERGE = 5; +223 private static final String TO_BE_LOADED = "to_be_loaded"; +224 /** +225 * Here is where hbase-1.x used to default the lock for hbck1. +226 * It puts in place a lock when it goes to write/make changes. +227 */ +228 @VisibleForTesting +229 public static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; +230 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; +231 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds +232 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds +233 // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. +234 // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for +235 // AlreadyBeingCreatedException which is implies timeout on this operations up to +236 // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). +237 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds +238 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5; +239 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds +240 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds +241 +242 /********************** +243 * Internal resources +244 **********************/ +245 private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName()); +246 private ClusterMetrics status; +247 private ClusterConnection connection; +248 private Admin admin; +249 private Table meta; +250 // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions +251 protected ExecutorService executor; +252 private long startMillis = EnvironmentEdgeManager.currentTime(); +253 private HFileCorruptionChecker hfcc; +254 private int retcode = 0; +255 private Path HBCK_LOCK_PATH; +256 private FSDataOutputStream hbckOutFd; +257 // This lock is to prevent cleanup of balancer resources twice between +258 // ShutdownHook and the main code. We cleanup only if the connect() is +259 // successful +260 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false); +261 +262 // Unsupported options in HBase 2.0+ +263 private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix", +264 "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans", +265 "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents", +266 "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge"); +267 +268 /*********** +269 * Options +270 ***********/ +271 private static boolean details = false; // do we display the full report +272 private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older +273 private static boolean forceExclusive = false; // only this hbck can modify HBase +274 private boolean fixAssignments = false; // fix assignment errors? +275 private boolean fixMeta = false; // fix meta errors? +276 private boolean checkHdfs = true; // load and check fs consistency? +277 private boolean fixHdfsHoles = false; // fix fs holes? +278 private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) +279 private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) +280 private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) +281 private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs +282 private boolean fixSplitParents = false; // fix lingering split parents +283 private boolean removeParents = false; // remove split parents +284 private boolean fixReferenceFiles = false; // fix lingering reference store file +285 private boolean fixHFileLinks = false; // fix lingering HFileLinks +286 private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows +287 private boolean fixReplication = false; // fix undeleted replication queues for removed peer +288 private boolean cleanReplicationBarrier = false; // clean replication barriers of a table +289 private boolean fixAny = false; // Set to true if any of the fix is required. +290 +291 // limit checking/fixes to listed tables, if empty attempt to check/fix all +292 // hbase:meta are always checked +293 private Set<TableName> tablesIncluded = new HashSet<>(); +294 private TableName cleanReplicationBarrierTable; +295 private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge +296 // maximum number of overlapping regions to sideline +297 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; +298 private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions +299 private Path sidelineDir = null; +300 +301 private boolean rerun = false; // if we tried to fix something, rerun hbck +302 private static boolean summary = false; // if we want to print less output +303 private boolean checkMetaOnly = false; +304 private boolean checkRegionBoundaries = false; +305 private boolean ignorePreCheckPermission = false; // if pre-check permission +306 +307 /********* +308 * State +309 *********/ +310 final private ErrorReporter errors; +311 int fixes = 0; 312 313 /** -314 * This map from Tablename -> TableInfo contains the structures necessary to -315 * detect table consistency problems (holes, dupes, overlaps). It is sorted -316 * to prevent dupes. -317 * -318 * If tablesIncluded is empty, this map contains all tables. -319 * Otherwise, it contains only meta tables and tables in tablesIncluded, -320 * unless checkMetaOnly is specified, in which case, it contains only -321 * the meta table -322 */ -323 private SortedMap<TableName, TableInfo> tablesInfo = new ConcurrentSkipListMap<>(); -324 -325 /** -326 * When initially looking at HDFS, we attempt to find any orphaned data. -327 */ -328 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>()); -329 -330 private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>(); -331 private Map<TableName, TableState> tableStates = new HashMap<>(); -332 private final RetryCounterFactory lockFileRetryCounterFactory; -333 private final RetryCounterFactory createZNodeRetryCounterFactory; -334 -335 private Map<TableName, Set<String>> skippedRegions = new HashMap<>(); -336 -337 private ZKWatcher zkw = null; -338 private String hbckEphemeralNodePath = null; -339 private boolean hbckZodeCreated = false; -340 -341 /** -342 * Constructor -343 * -344 * @param conf Configuration object -345 * @throws MasterNotRunningException if the master is not running -346 * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper -347 */ -348 public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException { -349 this(conf, createThreadPool(conf)); -350 } -351 -352 private static ExecutorService createThreadPool(Configuration conf) { -353 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); -354 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); -355 } -356 -357 /** -358 * Constructor -359 * -360 * @param conf -361 * Configuration object -362 * @throws MasterNotRunningException -363 * if the master is not running -364 * @throws ZooKeeperConnectionException -365 * if unable to connect to ZooKeeper -366 */ -367 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, -368 ZooKeeperConnectionException, IOException, ClassNotFoundException { -369 super(conf); -370 errors = getErrorReporter(getConf()); -371 this.executor = exec; -372 lockFileRetryCounterFactory = new RetryCounterFactory( -373 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), -374 getConf().getInt( -375 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), -376 getConf().getInt( -377 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); -378 createZNodeRetryCounterFactory = new RetryCounterFactory( -379 getConf().getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS), -380 getConf().getInt( -381 "hbase.hbck.createznode.attempt.sleep.interval", -382 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL), -383 getConf().getInt( -384 "hbase.hbck.createznode.attempt.maxsleeptime", -385 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME)); -386 zkw = createZooKeeperWatcher(); -387 } -388 -389 private class FileLockCallable implements Callable<FSDataOutputStream> { -390 RetryCounter retryCounter; -391 -392 public FileLockCallable(RetryCounter retryCounter) { -393 this.retryCounter = retryCounter; -394 } -395 @Override -396 public FSDataOutputStream call() throws IOException { -397 try { -398 FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); -399 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(), -400 HConstants.DATA_FILE_UMASK_KEY); -401 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY); -402 fs.mkdirs(tmpDir); -403 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE); -404 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms); -405 out.writeBytes(InetAddress.getLocalHost().toString()); -406 out.flush(); -407 return out; -408 } catch(RemoteException e) { -409 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){ -410 return null; -411 } else { -412 throw e; -413 } -414 } -415 } -416 -417 private FSDataOutputStream createFileWithRetries(final FileSystem fs, -418 final Path hbckLockFilePath, final FsPermission defaultPerms) -419 throws IOException { -420 -421 IOException exception = null; -422 do { -423 try { -424 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); -425 } catch (IOException ioe) { -426 LOG.info("Failed to create lock file " + hbckLockFilePath.getName() -427 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " -428 + retryCounter.getMaxAttempts()); -429 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), -430 ioe); -431 try { -432 exception = ioe; -433 retryCounter.sleepUntilNextRetry(); -434 } catch (InterruptedException ie) { -435 throw (InterruptedIOException) new InterruptedIOException( -436 "Can't create lock file " + hbckLockFilePath.getName()) -437 .initCause(ie); -438 } -439 } -440 } while (retryCounter.shouldRetry()); -441 -442 throw exception; -443 } -444 } -445 -446 /** -447 * This method maintains a lock using a file. If the creation fails we return null -448 * -449 * @return FSDataOutputStream object corresponding to the newly opened lock file -450 * @throws IOException if IO failure occurs -451 */ -452 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException { -453 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); -454 FileLockCallable callable = new FileLockCallable(retryCounter); -455 ExecutorService executor = Executors.newFixedThreadPool(1); -456 FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable); -457 executor.execute(futureTask); -458 final int timeoutInSeconds = getConf().getInt( -459 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT); -460 FSDataOutputStream stream = null; -461 try { -462 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS); -463 } catch (ExecutionException ee) { -464 LOG.warn("Encountered exception when opening lock file", ee); -465 } catch (InterruptedException ie) { -466 LOG.warn("Interrupted when opening lock file", ie); -467 Thread.currentThread().interrupt(); -468 } catch (TimeoutException exception) { -469 // took too long to obtain lock -470 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock"); -471 futureTask.cancel(true); -472 } finally { -473 executor.shutdownNow(); -474 } -475 return stream; -476 } -477 -478 private void unlockHbck() { -479 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) { -480 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); -481 do { -482 try { -483 IOUtils.closeQuietly(hbckOutFd); -484 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), -485 HBCK_LOCK_PATH, true); -486 LOG.info("Finishing hbck"); -487 return; -488 } catch (IOException ioe) { -489 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" -490 + (retryCounter.getAttemptTimes() + 1) + " of " -491 + retryCounter.getMaxAttempts()); -492 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe); -493 try { -494 retryCounter.sleepUntilNextRetry(); -495 } catch (InterruptedException ie) { -496 Thread.currentThread().interrupt(); -497 LOG.warn("Interrupted while deleting lock file" + -498 HBCK_LOCK_PATH); -499 return; -500 } -501 } -502 } while (retryCounter.shouldRetry()); -503 } -504 } -505 -506 /** -507 * To repair region consistency, one must call connect() in order to repair -508 * online state. -509 */ -510 public void connect() throws IOException { -511 -512 if (isExclusive()) { -513 // Grab the lock -514 hbckOutFd = checkAndMarkRunningHbck(); -515 if (hbckOutFd == null) { -516 setRetCode(-1); -517 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " + -518 "[If you are sure no other instance is running, delete the lock file " + -519 HBCK_LOCK_PATH + " and rerun the tool]"); -520 throw new IOException("Duplicate hbck - Abort"); -521 } -522 -523 // Make sure to cleanup the lock -524 hbckLockCleanup.set(true); -525 } -526 -527 -528 // Add a shutdown hook to this thread, in case user tries to -529 // kill the hbck with a ctrl-c, we want to cleanup the lock so that -530 // it is available for further calls -531 Runtime.getRuntime().addShutdownHook(new Thread() { -532 @Override -533 public void run() { -534 IOUtils.closeQuietly(HBaseFsck.this); -535 cleanupHbckZnode(); -536 unlockHbck(); -537 } -538 }); -539 -540 LOG.info("Launching hbck"); -541 -542 connection = (ClusterConnection)ConnectionFactory.createConnection(getConf()); -543 admin = connection.getAdmin(); -544 meta = connection.getTable(TableName.META_TABLE_NAME); -545 status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS, -546 Option.DEAD_SERVERS, Option.MASTER, Option.BACKUP_MASTERS, -547 Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION)); -548 } -549 -550 /** -551 * Get deployed regions according to the region servers. -552 */ -553 private void loadDeployedRegions() throws IOException, InterruptedException { -554 // From the master, get a list of all known live region servers -555 Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet(); -556 errors.print("Number of live region servers: " + regionServers.size()); -557 if (details) { -558 for (ServerName rsinfo: regionServers) { -559 errors.print(" " + rsinfo.getServerName()); -560 } -561 } -562 -563 // From the master, get a list of all dead region servers -564 Collection<ServerName> deadRegionServers = status.getDeadServerNames(); -565 errors.print("Number of dead region servers: " + deadRegionServers.size()); -566 if (details) { -567 for (ServerName name: deadRegionServers) { -568 errors.print(" " + name); -569 } +314 * This map contains the state of all hbck items. It maps from encoded region +315 * name to HbckInfo structure. The information contained in HbckInfo is used +316 * to detect and correct consistency (hdfs/meta/deployment) problems. +317 */ +318 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<>(); +319 // Empty regioninfo qualifiers in hbase:meta +320 private Set<Result> emptyRegionInfoQualifiers = new HashSet<>(); +321 +322 /** +323 * This map from Tablename -> TableInfo contains the structures necessary to +324 * detect table consistency problems (holes, dupes, overlaps). It is sorted +325 * to prevent dupes. +326 * +327 * If tablesIncluded is empty, this map contains all tables. +328 * Otherwise, it contains only meta tables and tables in tablesIncluded, +329 * unless checkMetaOnly is specified, in which case, it contains only +330 * the meta table +331 */ +332 private SortedMap<TableName, TableInfo> tablesInfo = new ConcurrentSkipListMap<>(); +333 +334 /** +335 * When initially looking at HDFS, we attempt to find any orphaned data. +336 */ +337 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>()); +338 +339 private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>(); +340 private Map<TableName, TableState> tableStates = new HashMap<>(); +341 private final RetryCounterFactory lockFileRetryCounterFactory; +342 private final RetryCounterFactory createZNodeRetryCounterFactory; +343 +344 private Map<TableName, Set<String>> skippedRegions = new HashMap<>(); +345 +346 private ZKWatcher zkw = null; +347 private String hbckEphemeralNodePath = null; +348 private boolean hbckZodeCreated = false; +349 +350 /** +351 * Constructor +352 * +353 * @param conf Configuration object +354 * @throws MasterNotRunningException if the master is not running +355 * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper +356 */ +357 public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException { +358 this(conf, createThreadPool(conf)); +359 } +360 +361 private static ExecutorService createThreadPool(Configuration conf) { +362 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); +363 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); +364 } +365 +366 /** +367 * Constructor +368 * +369 * @param conf +370 * Configuration object +371 * @throws MasterNotRunningException +372 * if the master is not running +373 * @throws ZooKeeperConnectionException +374 * if unable to connect to ZooKeeper +375 */ +376 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, +377 ZooKeeperConnectionException, IOException, ClassNotFoundException { +378 super(conf); +379 errors = getErrorReporter(getConf()); +380 this.executor = exec; +381 lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf()); +382 createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf()); +383 zkw = createZooKeeperWatcher(); +384 } +385 +386 /** +387 * @return A retry counter factory configured for retrying lock file creation. +388 */ +389 public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) { +390 return new RetryCounterFactory( +391 conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), +392 conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval", +393 DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), +394 conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime", +395 DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); +396 } +397 +398 /** +399 * @return A retry counter factory configured for retrying znode creation. +400 */ +401 private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) { +402 return new RetryCounterFactory( +403 conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS), +404 conf.getInt("hbase.hbck.createznode.attempt.sleep.interval", +405 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL), +406 conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime", +407 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME)); +408 } +409 +410 /** +411 * @return Return the tmp dir this tool writes too. +412 */ +413 @VisibleForTesting +414 public static Path getTmpDir(Configuration conf) throws IOException { +415 return new Path(FSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY); +416 } +417 +418 private static class FileLockCallable implements Callable<FSDataOutputStream> { +419 RetryCounter retryCounter; +420 private final Configuration conf; +421 private Path hbckLockPath = null; +422 +423 public FileLockCallable(Configuration conf, RetryCounter retryCounter) { +424 this.retryCounter = retryCounter; +425 this.conf = conf; +426 } +427 +428 /** +429 * @return Will be <code>null</code> unless you call {@link #call()} +430 */ +431 Path getHbckLockPath() { +432 return this.hbckLockPath; +433 } +434 +435 @Override +436 public FSDataOutputStream call() throws IOException { +437 try { +438 FileSystem fs = FSUtils.getCurrentFileSystem(this.conf); +439 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, this.conf, +440 HConstants.DATA_FILE_UMASK_KEY); +441 Path tmpDir = getTmpDir(conf); +442 this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE); +443 fs.mkdirs(tmpDir); +444 final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms); +445 out.writeBytes(InetAddress.getLocalHost().toString()); +446 // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file. +447 out.writeBytes(" Written by an hbase-2.x Master to block an " + +448 "attempt by an hbase-1.x HBCK tool making modification to state. " + +449 "See 'HBCK must match HBase server version' in the hbase refguide."); +450 out.flush(); +451 return out; +452 } catch(RemoteException e) { +453 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){ +454 return null; +455 } else { +456 throw e; +457 } +458 } +459 } +460 +461 private FSDataOutputStream createFileWithRetries(final FileSystem fs, +462 final Path hbckLockFilePath, final FsPermission defaultPerms) +463 throws IOException { +464 IOException exception = null; +465 do { +466 try { +467 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); +468 } catch (IOException ioe) { +469 LOG.info("Failed to create lock file " + hbckLockFilePath.getName() +470 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " +471 + retryCounter.getMaxAttempts()); +472 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), +473 ioe); +474 try { +475 exception = ioe; +476 retryCounter.sleepUntilNextRetry(); +477 } catch (InterruptedException ie) { +478 throw (InterruptedIOException) new InterruptedIOException( +479 "Can't create lock file " + hbckLockFilePath.getName()) +480 .initCause(ie); +481 } +482 } +483 } while (retryCounter.shouldRetry()); +484 +485 throw exception; +486 } +487 } +488 +489 /** +490 * This method maintains a lock using a file. If the creation fails we return null +491 * +492 * @return FSDataOutputStream object corresponding to the newly opened lock file +493 * @throws IOException if IO failure occurs +494 */ +495 public static Pair<Path, FSDataOutputStream> checkAndMarkRunningHbck(Configuration conf, +496 RetryCounter retryCounter) throws IOException { +497 FileLockCallable callable = new FileLockCallable(conf, retryCounter); +498 ExecutorService executor = Executors.newFixedThreadPool(1); +499 FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable); +500 executor.execute(futureTask); +501 final int timeoutInSeconds = conf.getInt( +502 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT); +503 FSDataOutputStream stream = null; +504 try { +505 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS); +506 } catch (ExecutionException ee) { +507 LOG.warn("Encountered exception when opening lock file", ee); +508 } catch (InterruptedException ie) { +509 LOG.warn("Interrupted when opening lock file", ie); +510 Thread.currentThread().interrupt(); +511 } catch (TimeoutException exception) { +512 // took too long to obtain lock +513 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock"); +514 futureTask.cancel(true); +515 } finally { +516 executor.shutdownNow(); +517 } +518 return new Pair<Path, FSDataOutputStream>(callable.getHbckLockPath(), stream); +519 } +520 +521 private void unlockHbck() { +522 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) { +523 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); +524 do { +525 try { +526 IOUtils.closeQuietly(hbckOutFd); +527 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true); +528 LOG.info("Finishing hbck"); +529 return; +530 } catch (IOException ioe) { +531 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" +532 + (retryCounter.getAttemptTimes() + 1) + " of " +533 + retryCounter.getMaxAttempts()); +534 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe); +535 try { +536 retryCounter.sleepUntilNextRetry(); +537 } catch (InterruptedException ie) { +538 Thread.currentThread().interrupt(); +539 LOG.warn("Interrupted while deleting lock file" + +540 HBCK_LOCK_PATH); +541 return; +542 } +543 } +544 } while (retryCounter.shouldRetry()); +545 } +546 } +547 +548 /** +549 * To repair region consistency, one must call connect() in order to repair +550 * online state. +551 */ +552 public void connect() throws IOException { +553 +554 if (isExclusive()) { +555 // Grab the lock +556 Pair<Path, FSD