Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A2DDE200CE0 for ; Fri, 25 Aug 2017 17:10:06 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A15B916CBA7; Fri, 25 Aug 2017 15:10:06 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 68D5C16CB83 for ; Fri, 25 Aug 2017 17:10:04 +0200 (CEST) Received: (qmail 74384 invoked by uid 500); 25 Aug 2017 15:09:59 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 71935 invoked by uid 99); 25 Aug 2017 15:09:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Aug 2017 15:09:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1EDE2F5F31; Fri, 25 Aug 2017 15:09:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Fri, 25 Aug 2017 15:10:00 -0000 Message-Id: <9bbca270fa06424c9c6bcd727e85a8f8@git.apache.org> In-Reply-To: <5873fcae1a1f4e888c9c2af2d564a58a@git.apache.org> References: <5873fcae1a1f4e888c9c2af2d564a58a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/51] [partial] hbase-site git commit: Published site at . archived-at: Fri, 25 Aug 2017 15:10:06 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/6b5143ed/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.TableInfo.HDFSIntegrityFixer.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.TableInfo.HDFSIntegrityFixer.html b/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.TableInfo.HDFSIntegrityFixer.html index f5186b5..43afb13 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.TableInfo.HDFSIntegrityFixer.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/util/HBaseFsck.TableInfo.HDFSIntegrityFixer.html @@ -36,5085 +36,5084 @@ 028import org.apache.hadoop.hbase.shaded.com.google.common.collect.Multimap; 029import org.apache.hadoop.hbase.shaded.com.google.common.collect.Ordering; 030import org.apache.hadoop.hbase.shaded.com.google.common.collect.TreeMultimap; -031import com.google.protobuf.ServiceException; -032 -033import java.io.Closeable; -034import java.io.FileNotFoundException; -035import java.io.IOException; -036import java.io.InterruptedIOException; -037import java.io.PrintWriter; -038import java.io.StringWriter; -039import java.net.InetAddress; -040import java.net.URI; -041import java.util.ArrayList; -042import java.util.Arrays; -043import java.util.Collection; -044import java.util.Collections; -045import java.util.Comparator; -046import java.util.HashMap; -047import java.util.HashSet; -048import java.util.Iterator; -049import java.util.List; -050import java.util.Locale; -051import java.util.Map; -052import java.util.Map.Entry; -053import java.util.Set; -054import java.util.SortedMap; -055import java.util.SortedSet; -056import java.util.TreeMap; -057import java.util.TreeSet; -058import java.util.Vector; -059import java.util.concurrent.Callable; -060import java.util.concurrent.ConcurrentSkipListMap; -061import java.util.concurrent.ExecutionException; -062import java.util.concurrent.ExecutorService; -063import java.util.concurrent.Executors; -064import java.util.concurrent.Future; -065import java.util.concurrent.FutureTask; -066import java.util.concurrent.ScheduledThreadPoolExecutor; -067import java.util.concurrent.TimeUnit; -068import java.util.concurrent.TimeoutException; -069import java.util.concurrent.atomic.AtomicBoolean; -070import java.util.concurrent.atomic.AtomicInteger; -071 -072import org.apache.commons.io.IOUtils; -073import org.apache.commons.lang.RandomStringUtils; -074import org.apache.commons.lang.StringUtils; -075import org.apache.commons.logging.Log; -076import org.apache.commons.logging.LogFactory; -077import org.apache.hadoop.conf.Configuration; -078import org.apache.hadoop.conf.Configured; -079import org.apache.hadoop.fs.FSDataOutputStream; -080import org.apache.hadoop.fs.FileStatus; -081import org.apache.hadoop.fs.FileSystem; -082import org.apache.hadoop.fs.Path; -083import org.apache.hadoop.fs.permission.FsAction; -084import org.apache.hadoop.fs.permission.FsPermission; -085import org.apache.hadoop.hbase.Abortable; -086import org.apache.hadoop.hbase.Cell; -087import org.apache.hadoop.hbase.CellUtil; -088import org.apache.hadoop.hbase.ClusterStatus; -089import org.apache.hadoop.hbase.HBaseConfiguration; -090import org.apache.hadoop.hbase.HBaseInterfaceAudience; -091import org.apache.hadoop.hbase.HConstants; -092import org.apache.hadoop.hbase.HRegionInfo; -093import org.apache.hadoop.hbase.HRegionLocation; -094import org.apache.hadoop.hbase.KeyValue; -095import org.apache.hadoop.hbase.MasterNotRunningException; -096import org.apache.hadoop.hbase.MetaTableAccessor; -097import org.apache.hadoop.hbase.RegionLocations; -098import org.apache.hadoop.hbase.ServerName; -099import org.apache.hadoop.hbase.TableName; -100import org.apache.hadoop.hbase.ZooKeeperConnectionException; -101import org.apache.hadoop.hbase.classification.InterfaceAudience; -102import org.apache.hadoop.hbase.classification.InterfaceStability; -103import org.apache.hadoop.hbase.client.Admin; -104import org.apache.hadoop.hbase.client.ClusterConnection; -105import org.apache.hadoop.hbase.client.Connection; -106import org.apache.hadoop.hbase.client.ConnectionFactory; -107import org.apache.hadoop.hbase.client.Delete; -108import org.apache.hadoop.hbase.client.Get; -109import org.apache.hadoop.hbase.client.Put; -110import org.apache.hadoop.hbase.client.RegionReplicaUtil; -111import org.apache.hadoop.hbase.client.Result; -112import org.apache.hadoop.hbase.client.RowMutations; -113import org.apache.hadoop.hbase.client.Table; -114import org.apache.hadoop.hbase.client.TableState; -115import org.apache.hadoop.hbase.io.FileLink; -116import org.apache.hadoop.hbase.io.HFileLink; -117import org.apache.hadoop.hbase.io.hfile.CacheConfig; -118import org.apache.hadoop.hbase.io.hfile.HFile; -119import org.apache.hadoop.hbase.master.MasterFileSystem; -120import org.apache.hadoop.hbase.master.RegionState; -121import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -122import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface; -123import org.apache.hadoop.hbase.regionserver.HRegion; -124import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -125import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -126import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; -127import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; -128import org.apache.hadoop.hbase.security.AccessDeniedException; -129import org.apache.hadoop.hbase.security.UserProvider; -130import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; -131import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; -132import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; -133import org.apache.hadoop.hbase.util.hbck.ReplicationChecker; -134import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; -135import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl; -136import org.apache.hadoop.hbase.wal.WAL; -137import org.apache.hadoop.hbase.wal.WALFactory; -138import org.apache.hadoop.hbase.wal.WALSplitter; -139import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -140import org.apache.hadoop.hbase.zookeeper.ZKUtil; -141import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -142import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; -143import org.apache.hadoop.ipc.RemoteException; -144import org.apache.hadoop.security.UserGroupInformation; -145import org.apache.hadoop.util.ReflectionUtils; -146import org.apache.hadoop.util.Tool; -147import org.apache.hadoop.util.ToolRunner; -148import org.apache.zookeeper.KeeperException; -149 -150/** -151 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and -152 * table integrity problems in a corrupted HBase. -153 * -154 * Region consistency checks verify that hbase:meta, region deployment on region -155 * servers and the state of data in HDFS (.regioninfo files) all are in -156 * accordance. -157 * -158 * Table integrity checks verify that all possible row keys resolve to exactly -159 * one region of a table. This means there are no individual degenerate -160 * or backwards regions; no holes between regions; and that there are no -161 * overlapping regions. -162 * -163 * The general repair strategy works in two phases: -164 * <ol> -165 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions) -166 * <li> Repair Region Consistency with hbase:meta and assignments -167 * </ol> -168 * -169 * For table integrity repairs, the tables' region directories are scanned -170 * for .regioninfo files. Each table's integrity is then verified. If there -171 * are any orphan regions (regions with no .regioninfo files) or holes, new -172 * regions are fabricated. Backwards regions are sidelined as well as empty -173 * degenerate (endkey==startkey) regions. If there are any overlapping regions, -174 * a new region is created and all data is merged into the new region. -175 * -176 * Table integrity repairs deal solely with HDFS and could potentially be done -177 * offline -- the hbase region servers or master do not need to be running. -178 * This phase can eventually be used to completely reconstruct the hbase:meta table in -179 * an offline fashion. -180 * -181 * Region consistency requires three conditions -- 1) valid .regioninfo file -182 * present in an HDFS region dir, 2) valid row with .regioninfo data in META, -183 * and 3) a region is deployed only at the regionserver that was assigned to -184 * with proper state in the master. -185 * -186 * Region consistency repairs require hbase to be online so that hbck can -187 * contact the HBase master and region servers. The hbck#connect() method must -188 * first be called successfully. Much of the region consistency information -189 * is transient and less risky to repair. -190 * -191 * If hbck is run from the command line, there are a handful of arguments that -192 * can be used to limit the kinds of repairs hbck will do. See the code in -193 * {@link #printUsageAndExit()} for more details. -194 */ -195@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -196@InterfaceStability.Evolving -197public class HBaseFsck extends Configured implements Closeable { -198 public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute -199 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; -200 private static final int MAX_NUM_THREADS = 50; // #threads to contact regions -201 private static boolean rsSupportsOffline = true; -202 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; -203 private static final int DEFAULT_MAX_MERGE = 5; -204 private static final String TO_BE_LOADED = "to_be_loaded"; -205 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; -206 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; -207 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds -208 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds -209 // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. -210 // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for -211 // AlreadyBeingCreatedException which is implies timeout on this operations up to -212 // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). -213 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds -214 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5; -215 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds -216 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds -217 -218 /********************** -219 * Internal resources -220 **********************/ -221 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName()); -222 private ClusterStatus status; -223 private ClusterConnection connection; -224 private Admin admin; -225 private Table meta; -226 // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions -227 protected ExecutorService executor; -228 private long startMillis = EnvironmentEdgeManager.currentTime(); -229 private HFileCorruptionChecker hfcc; -230 private int retcode = 0; -231 private Path HBCK_LOCK_PATH; -232 private FSDataOutputStream hbckOutFd; -233 // This lock is to prevent cleanup of balancer resources twice between -234 // ShutdownHook and the main code. We cleanup only if the connect() is -235 // successful -236 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false); -237 -238 /*********** -239 * Options -240 ***********/ -241 private static boolean details = false; // do we display the full report -242 private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older -243 private static boolean forceExclusive = false; // only this hbck can modify HBase -244 private boolean fixAssignments = false; // fix assignment errors? -245 private boolean fixMeta = false; // fix meta errors? -246 private boolean checkHdfs = true; // load and check fs consistency? -247 private boolean fixHdfsHoles = false; // fix fs holes? -248 private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) -249 private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) -250 private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) -251 private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs -252 private boolean fixSplitParents = false; // fix lingering split parents -253 private boolean removeParents = false; // remove split parents -254 private boolean fixReferenceFiles = false; // fix lingering reference store file -255 private boolean fixHFileLinks = false; // fix lingering HFileLinks -256 private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows -257 private boolean fixReplication = false; // fix undeleted replication queues for removed peer -258 private boolean fixAny = false; // Set to true if any of the fix is required. -259 -260 // limit checking/fixes to listed tables, if empty attempt to check/fix all -261 // hbase:meta are always checked -262 private Set<TableName> tablesIncluded = new HashSet<>(); -263 private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge -264 // maximum number of overlapping regions to sideline -265 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; -266 private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions -267 private Path sidelineDir = null; -268 -269 private boolean rerun = false; // if we tried to fix something, rerun hbck -270 private static boolean summary = false; // if we want to print less output -271 private boolean checkMetaOnly = false; -272 private boolean checkRegionBoundaries = false; -273 private boolean ignorePreCheckPermission = false; // if pre-check permission -274 -275 /********* -276 * State -277 *********/ -278 final private ErrorReporter errors; -279 int fixes = 0; -280 -281 /** -282 * This map contains the state of all hbck items. It maps from encoded region -283 * name to HbckInfo structure. The information contained in HbckInfo is used -284 * to detect and correct consistency (hdfs/meta/deployment) problems. -285 */ -286 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<>(); -287 // Empty regioninfo qualifiers in hbase:meta -288 private Set<Result> emptyRegionInfoQualifiers = new HashSet<>(); -289 -290 /** -291 * This map from Tablename -> TableInfo contains the structures necessary to -292 * detect table consistency problems (holes, dupes, overlaps). It is sorted -293 * to prevent dupes. -294 * -295 * If tablesIncluded is empty, this map contains all tables. -296 * Otherwise, it contains only meta tables and tables in tablesIncluded, -297 * unless checkMetaOnly is specified, in which case, it contains only -298 * the meta table -299 */ -300 private SortedMap<TableName, TableInfo> tablesInfo = new ConcurrentSkipListMap<>(); -301 -302 /** -303 * When initially looking at HDFS, we attempt to find any orphaned data. -304 */ -305 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>()); -306 -307 private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>(); -308 private Map<TableName, TableState> tableStates = new HashMap<>(); -309 private final RetryCounterFactory lockFileRetryCounterFactory; -310 private final RetryCounterFactory createZNodeRetryCounterFactory; -311 -312 private Map<TableName, Set<String>> skippedRegions = new HashMap<>(); -313 -314 private ZooKeeperWatcher zkw = null; -315 private String hbckEphemeralNodePath = null; -316 private boolean hbckZodeCreated = false; -317 -318 /** -319 * Constructor -320 * -321 * @param conf Configuration object -322 * @throws MasterNotRunningException if the master is not running -323 * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper -324 */ -325 public HBaseFsck(Configuration conf) throws MasterNotRunningException, -326 ZooKeeperConnectionException, IOException, ClassNotFoundException { -327 this(conf, createThreadPool(conf)); -328 } -329 -330 private static ExecutorService createThreadPool(Configuration conf) { -331 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); -332 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); -333 } -334 -335 /** -336 * Constructor -337 * -338 * @param conf -339 * Configuration object -340 * @throws MasterNotRunningException -341 * if the master is not running -342 * @throws ZooKeeperConnectionException -343 * if unable to connect to ZooKeeper -344 */ -345 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, -346 ZooKeeperConnectionException, IOException, ClassNotFoundException { -347 super(conf); -348 errors = getErrorReporter(getConf()); -349 this.executor = exec; -350 lockFileRetryCounterFactory = new RetryCounterFactory( -351 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), -352 getConf().getInt( -353 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), -354 getConf().getInt( -355 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); -356 createZNodeRetryCounterFactory = new RetryCounterFactory( -357 getConf().getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS), -358 getConf().getInt( -359 "hbase.hbck.createznode.attempt.sleep.interval", -360 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL), -361 getConf().getInt( -362 "hbase.hbck.createznode.attempt.maxsleeptime", -363 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME)); -364 zkw = createZooKeeperWatcher(); -365 } -366 -367 private class FileLockCallable implements Callable<FSDataOutputStream> { -368 RetryCounter retryCounter; -369 -370 public FileLockCallable(RetryCounter retryCounter) { -371 this.retryCounter = retryCounter; -372 } -373 @Override -374 public FSDataOutputStream call() throws IOException { -375 try { -376 FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); -377 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(), -378 HConstants.DATA_FILE_UMASK_KEY); -379 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY); -380 fs.mkdirs(tmpDir); -381 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE); -382 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms); -383 out.writeBytes(InetAddress.getLocalHost().toString()); -384 out.flush(); -385 return out; -386 } catch(RemoteException e) { -387 if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){ -388 return null; -389 } else { -390 throw e; -391 } -392 } -393 } -394 -395 private FSDataOutputStream createFileWithRetries(final FileSystem fs, -396 final Path hbckLockFilePath, final FsPermission defaultPerms) -397 throws IOException { -398 -399 IOException exception = null; -400 do { -401 try { -402 return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); -403 } catch (IOException ioe) { -404 LOG.info("Failed to create lock file " + hbckLockFilePath.getName() -405 + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " -406 + retryCounter.getMaxAttempts()); -407 LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), -408 ioe); -409 try { -410 exception = ioe; -411 retryCounter.sleepUntilNextRetry(); -412 } catch (InterruptedException ie) { -413 throw (InterruptedIOException) new InterruptedIOException( -414 "Can't create lock file " + hbckLockFilePath.getName()) -415 .initCause(ie); -416 } -417 } -418 } while (retryCounter.shouldRetry()); -419 -420 throw exception; -421 } -422 } -423 -424 /** -425 * This method maintains a lock using a file. If the creation fails we return null -426 * -427 * @return FSDataOutputStream object corresponding to the newly opened lock file -428 * @throws IOException if IO failure occurs -429 */ -430 private FSDataOutputStream checkAndMarkRunningHbck() throws IOException { -431 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); -432 FileLockCallable callable = new FileLockCallable(retryCounter); -433 ExecutorService executor = Executors.newFixedThreadPool(1); -434 FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable); -435 executor.execute(futureTask); -436 final int timeoutInSeconds = getConf().getInt( -437 "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT); -438 FSDataOutputStream stream = null; -439 try { -440 stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS); -441 } catch (ExecutionException ee) { -442 LOG.warn("Encountered exception when opening lock file", ee); -443 } catch (InterruptedException ie) { -444 LOG.warn("Interrupted when opening lock file", ie); -445 Thread.currentThread().interrupt(); -446 } catch (TimeoutException exception) { -447 // took too long to obtain lock -448 LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock"); -449 futureTask.cancel(true); -450 } finally { -451 executor.shutdownNow(); -452 } -453 return stream; -454 } -455 -456 private void unlockHbck() { -457 if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) { -458 RetryCounter retryCounter = lockFileRetryCounterFactory.create(); -459 do { -460 try { -461 IOUtils.closeQuietly(hbckOutFd); -462 FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), -463 HBCK_LOCK_PATH, true); -464 LOG.info("Finishing hbck"); -465 return; -466 } catch (IOException ioe) { -467 LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" -468 + (retryCounter.getAttemptTimes() + 1) + " of " -469 + retryCounter.getMaxAttempts()); -470 LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe); -471 try { -472 retryCounter.sleepUntilNextRetry(); -473 } catch (InterruptedException ie) { -474 Thread.currentThread().interrupt(); -475 LOG.warn("Interrupted while deleting lock file" + -476 HBCK_LOCK_PATH); -477 return; -478 } -479 } -480 } while (retryCounter.shouldRetry()); -481 } -482 } -483 -484 /** -485 * To repair region consistency, one must call connect() in order to repair -486 * online state. -487 */ -488 public void connect() throws IOException { -489 -490 if (isExclusive()) { -491 // Grab the lock -492 hbckOutFd = checkAndMarkRunningHbck(); -493 if (hbckOutFd == null) { -494 setRetCode(-1); -495 LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " + -496 "[If you are sure no other instance is running, delete the lock file " + -497 HBCK_LOCK_PATH + " and rerun the tool]"); -498 throw new IOException("Duplicate hbck - Abort"); -499 } -500 -501 // Make sure to cleanup the lock -502 hbckLockCleanup.set(true); -503 } +031 +032import java.io.Closeable; +033import java.io.FileNotFoundException; +034import java.io.IOException; +035import java.io.InterruptedIOException; +036import java.io.PrintWriter; +037import java.io.StringWriter; +038import java.net.InetAddress; +039import java.net.URI; +040import java.util.ArrayList; +041import java.util.Arrays; +042import java.util.Collection; +043import java.util.Collections; +044import java.util.Comparator; +045import java.util.HashMap; +046import java.util.HashSet; +047import java.util.Iterator; +048import java.util.List; +049import java.util.Locale; +050import java.util.Map; +051import java.util.Map.Entry; +052import java.util.Set; +053import java.util.SortedMap; +054import java.util.SortedSet; +055import java.util.TreeMap; +056import java.util.TreeSet; +057import java.util.Vector; +058import java.util.concurrent.Callable; +059import java.util.concurrent.ConcurrentSkipListMap; +060import java.util.concurrent.ExecutionException; +061import java.util.concurrent.ExecutorService; +062import java.util.concurrent.Executors; +063import java.util.concurrent.Future; +064import java.util.concurrent.FutureTask; +065import java.util.concurrent.ScheduledThreadPoolExecutor; +066import java.util.concurrent.TimeUnit; +067import java.util.concurrent.TimeoutException; +068import java.util.concurrent.atomic.AtomicBoolean; +069import java.util.concurrent.atomic.AtomicInteger; +070 +071import org.apache.commons.io.IOUtils; +072import org.apache.commons.lang.RandomStringUtils; +073import org.apache.commons.lang.StringUtils; +074import org.apache.commons.logging.Log; +075import org.apache.commons.logging.LogFactory; +076import org.apache.hadoop.conf.Configuration; +077import org.apache.hadoop.conf.Configured; +078import org.apache.hadoop.fs.FSDataOutputStream; +079import org.apache.hadoop.fs.FileStatus; +080import org.apache.hadoop.fs.FileSystem; +081import org.apache.hadoop.fs.Path; +082import org.apache.hadoop.fs.permission.FsAction; +083import org.apache.hadoop.fs.permission.FsPermission; +084import org.apache.hadoop.hbase.Abortable; +085import org.apache.hadoop.hbase.Cell; +086import org.apache.hadoop.hbase.CellUtil; +087import org.apache.hadoop.hbase.ClusterStatus; +088import org.apache.hadoop.hbase.HBaseConfiguration; +089import org.apache.hadoop.hbase.HBaseInterfaceAudience; +090import org.apache.hadoop.hbase.HConstants; +091import org.apache.hadoop.hbase.HRegionInfo; +092import org.apache.hadoop.hbase.HRegionLocation; +093import org.apache.hadoop.hbase.KeyValue; +094import org.apache.hadoop.hbase.MasterNotRunningException; +095import org.apache.hadoop.hbase.MetaTableAccessor; +096import org.apache.hadoop.hbase.RegionLocations; +097import org.apache.hadoop.hbase.ServerName; +098import org.apache.hadoop.hbase.TableName; +099import org.apache.hadoop.hbase.ZooKeeperConnectionException; +100import org.apache.hadoop.hbase.classification.InterfaceAudience; +101import org.apache.hadoop.hbase.classification.InterfaceStability; +102import org.apache.hadoop.hbase.client.Admin; +103import org.apache.hadoop.hbase.client.ClusterConnection; +104import org.apache.hadoop.hbase.client.Connection; +105import org.apache.hadoop.hbase.client.ConnectionFactory; +106import org.apache.hadoop.hbase.client.Delete; +107import org.apache.hadoop.hbase.client.Get; +108import org.apache.hadoop.hbase.client.Put; +109import org.apache.hadoop.hbase.client.RegionReplicaUtil; +110import org.apache.hadoop.hbase.client.Result; +111import org.apache.hadoop.hbase.client.RowMutations; +112import org.apache.hadoop.hbase.client.Table; +113import org.apache.hadoop.hbase.client.TableState; +114import org.apache.hadoop.hbase.io.FileLink; +115import org.apache.hadoop.hbase.io.HFileLink; +116import org.apache.hadoop.hbase.io.hfile.CacheConfig; +117import org.apache.hadoop.hbase.io.hfile.HFile; +118import org.apache.hadoop.hbase.master.MasterFileSystem; +119import org.apache.hadoop.hbase.master.RegionState; +120import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +121import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface; +122import org.apache.hadoop.hbase.regionserver.HRegion; +123import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +124import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +125import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; +126import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +127import org.apache.hadoop.hbase.security.AccessDeniedException; +128import org.apache.hadoop.hbase.security.UserProvider; +129import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator; +130import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; +131import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; +132import org.apache.hadoop.hbase.util.hbck.ReplicationChecker; +133import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; +134import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl; +135import org.apache.hadoop.hbase.wal.WAL; +136import org.apache.hadoop.hbase.wal.WALFactory; +137import org.apache.hadoop.hbase.wal.WALSplitter; +138import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +139import org.apache.hadoop.hbase.zookeeper.ZKUtil; +140import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +141import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; +142import org.apache.hadoop.ipc.RemoteException; +143import org.apache.hadoop.security.UserGroupInformation; +144import org.apache.hadoop.util.ReflectionUtils; +145import org.apache.hadoop.util.Tool; +146import org.apache.hadoop.util.ToolRunner; +147import org.apache.zookeeper.KeeperException; +148 +149/** +150 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and +151 * table integrity problems in a corrupted HBase. +152 * +153 * Region consistency checks verify that hbase:meta, region deployment on region +154 * servers and the state of data in HDFS (.regioninfo files) all are in +155 * accordance. +156 * +157 * Table integrity checks verify that all possible row keys resolve to exactly +158 * one region of a table. This means there are no individual degenerate +159 * or backwards regions; no holes between regions; and that there are no +160 * overlapping regions. +161 * +162 * The general repair strategy works in two phases: +163 * <ol> +164 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions) +165 * <li> Repair Region Consistency with hbase:meta and assignments +166 * </ol> +167 * +168 * For table integrity repairs, the tables' region directories are scanned +169 * for .regioninfo files. Each table's integrity is then verified. If there +170 * are any orphan regions (regions with no .regioninfo files) or holes, new +171 * regions are fabricated. Backwards regions are sidelined as well as empty +172 * degenerate (endkey==startkey) regions. If there are any overlapping regions, +173 * a new region is created and all data is merged into the new region. +174 * +175 * Table integrity repairs deal solely with HDFS and could potentially be done +176 * offline -- the hbase region servers or master do not need to be running. +177 * This phase can eventually be used to completely reconstruct the hbase:meta table in +178 * an offline fashion. +179 * +180 * Region consistency requires three conditions -- 1) valid .regioninfo file +181 * present in an HDFS region dir, 2) valid row with .regioninfo data in META, +182 * and 3) a region is deployed only at the regionserver that was assigned to +183 * with proper state in the master. +184 * +185 * Region consistency repairs require hbase to be online so that hbck can +186 * contact the HBase master and region servers. The hbck#connect() method must +187 * first be called successfully. Much of the region consistency information +188 * is transient and less risky to repair. +189 * +190 * If hbck is run from the command line, there are a handful of arguments that +191 * can be used to limit the kinds of repairs hbck will do. See the code in +192 * {@link #printUsageAndExit()} for more details. +193 */ +194@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +195@InterfaceStability.Evolving +196public class HBaseFsck extends Configured implements Closeable { +197 public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute +198 public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000; +199 private static final int MAX_NUM_THREADS = 50; // #threads to contact regions +200 private static boolean rsSupportsOffline = true; +201 private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2; +202 private static final int DEFAULT_MAX_MERGE = 5; +203 private static final String TO_BE_LOADED = "to_be_loaded"; +204 private static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; +205 private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; +206 private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds +207 private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds +208 // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. +209 // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for +210 // AlreadyBeingCreatedException which is implies timeout on this operations up to +211 // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). +212 private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds +213 private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5; +214 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds +215 private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds +216 +217 /********************** +218 * Internal resources +219 **********************/ +220 private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName()); +221 private ClusterStatus status; +222 private ClusterConnection connection; +223 private Admin admin; +224 private Table meta; +225 // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions +226 protected ExecutorService executor; +227 private long startMillis = EnvironmentEdgeManager.currentTime(); +228 private HFileCorruptionChecker hfcc; +229 private int retcode = 0; +230 private Path HBCK_LOCK_PATH; +231 private FSDataOutputStream hbckOutFd; +232 // This lock is to prevent cleanup of balancer resources twice between +233 // ShutdownHook and the main code. We cleanup only if the connect() is +234 // successful +235 private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false); +236 +237 /*********** +238 * Options +239 ***********/ +240 private static boolean details = false; // do we display the full report +241 private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older +242 private static boolean forceExclusive = false; // only this hbck can modify HBase +243 private boolean fixAssignments = false; // fix assignment errors? +244 private boolean fixMeta = false; // fix meta errors? +245 private boolean checkHdfs = true; // load and check fs consistency? +246 private boolean fixHdfsHoles = false; // fix fs holes? +247 private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) +248 private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) +249 private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) +250 private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs +251 private boolean fixSplitParents = false; // fix lingering split parents +252 private boolean removeParents = false; // remove split parents +253 private boolean fixReferenceFiles = false; // fix lingering reference store file +254 private boolean fixHFileLinks = false; // fix lingering HFileLinks +255 private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows +256 private boolean fixReplication = false; // fix undeleted replication queues for removed peer +257 private boolean fixAny = false; // Set to true if any of the fix is required. +258 +259 // limit checking/fixes to listed tables, if empty attempt to check/fix all +260 // hbase:meta are always checked +261 private Set<TableName> tablesIncluded = new HashSet<>(); +262 private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge +263 // maximum number of overlapping regions to sideline +264 private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; +265 private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions +266 private Path sidelineDir = null; +267 +268 private boolean rerun = false; // if we tried to fix something, rerun hbck +269 private static boolean summary = false; // if we want to print less output +270 private boolean checkMetaOnly = false; +271 private boolean checkRegionBoundaries = false; +272 private boolean ignorePreCheckPermission = false; // if pre-check permission +273 +274 /********* +275 * State +276 *********/ +277 final private ErrorReporter errors; +278 int fixes = 0; +279 +280 /** +281 * This map contains the state of all hbck items. It maps from encoded region +282 * name to HbckInfo structure. The information contained in HbckInfo is used +283 * to detect and correct consistency (hdfs/meta/deployment) problems. +284 */ +285 private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<>(); +286 // Empty regioninfo qualifiers in hbase:meta +287 private Set<Result> emptyRegionInfoQualifiers = new HashSet<>(); +288 +289 /** +290 * This map from Tablename -> TableInfo contains the structures necessary to +291 * detect table consistency problems (holes, dupes, overlaps). It is sorted +292 * to prevent dupes. +293 * +294 * If tablesIncluded is empty, this map contains all tables. +295 * Otherwise, it contains only meta tables and tables in tablesIncluded, +296 * unless checkMetaOnly is specified, in which case, it contains only +297 * the meta table +298 */ +299 private SortedMap<TableName, TableInfo> tablesInfo = new ConcurrentSkipListMap<>(); +300 +301 /** +302 * When initially looking at HDFS, we attempt to find any orphaned data. +303 */ +304 private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>()); +305 +306 private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>(); +307 private Map<TableName, TableState> tableStates = new HashMap<>(); +308 private final RetryCounterFactory lockFileRetryCounterFactory; +309 private final RetryCounterFactory createZNodeRetryCounterFactory; +310 +311 private Map<TableName, Set<String>> skippedRegions = new HashMap<>(); +312 +313 private ZooKeeperWatcher zkw = null; +314 private String hbckEphemeralNodePath = null; +315 private boolean hbckZodeCreated = false; +316 +317 /** +318 * Constructor +319 * +320 * @param conf Configuration object +321 * @throws MasterNotRunningException if the master is not running +322 * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper +323 */ +324 public HBaseFsck(Configuration conf) throws MasterNotRunningException, +325 ZooKeeperConnectionException, IOException, ClassNotFoundException { +326 this(conf, createThreadPool(conf)); +327 } +328 +329 private static ExecutorService createThreadPool(Configuration conf) { +330 int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); +331 return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); +332 } +333 +334 /** +335 * Constructor +336 * +337 * @param conf +338 * Configuration object +339 * @throws MasterNotRunningException +340 * if the master is not running +341 * @throws ZooKeeperConnectionException +342 * if unable to connect to ZooKeeper +343 */ +344 public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException, +345 ZooKeeperConnectionException, IOException, ClassNotFoundException { +346 super(conf); +347 errors = getErrorReporter(getConf()); +348 this.executor = exec; +349 lockFileRetryCounterFactory = new RetryCounterFactory( +350 getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), +351 getConf().getInt( +352 "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), +353 getConf().getInt( +354 "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); +355 createZNodeRetryCounterFactory = new RetryCounterFactory( +356 getConf().getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS), +357 getConf().getInt( +358 "hbase.hbck.createznode.attempt.sleep.interval", +359 DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL), +360 getConf().getInt( +361 "hbase.hbck.createznode.attempt.maxsleeptime", +362 DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME)); +363 zkw = createZooKeeperWatcher(); +364 } +365 +366 private class FileLockCallable implements Callable<FSDataOutputStream> { +367 RetryCounter retryCounter; +368 +369 public FileLockCallable(RetryCounter retryCounter) { +370 this.retryCounter = retryCounter; +371 } +372 @Override +373 public FSDataOutputStream call() throws IOException { +374 try { +375 FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); +376 FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(), +377 HConstants.DATA_FILE_UMASK_KEY); +378 Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY); +379 fs.mkdirs(tmpDir); +380 HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE); +381 final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms); +382 out.writeBytes(InetAddress.getLocalHost().toString());<