Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9D270200CB5 for ; Tue, 27 Jun 2017 17:01:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 9BFA8160BC6; Tue, 27 Jun 2017 15:01:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A0BD9160BDC for ; Tue, 27 Jun 2017 17:01:34 +0200 (CEST) Received: (qmail 31042 invoked by uid 500); 27 Jun 2017 15:01:26 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 26032 invoked by uid 99); 27 Jun 2017 15:01:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 27 Jun 2017 15:01:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1EF50F323C; Tue, 27 Jun 2017 15:01:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 27 Jun 2017 15:01:55 -0000 Message-Id: <7bf27a4e8b644a89bcabdd24c0ae2beb@git.apache.org> In-Reply-To: <226601fc29d5464ba43fac0105ae50b9@git.apache.org> References: <226601fc29d5464ba43fac0105ae50b9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [41/51] [partial] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Tue, 27 Jun 2017 15:01:36 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8e3b63ca/devapidocs/src-html/org/apache/hadoop/hbase/AsyncMetaTableAccessor.MetaTableRawScanResultConsumer.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/AsyncMetaTableAccessor.MetaTableRawScanResultConsumer.html b/devapidocs/src-html/org/apache/hadoop/hbase/AsyncMetaTableAccessor.MetaTableRawScanResultConsumer.html index a0256ee..5be1ff9 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/AsyncMetaTableAccessor.MetaTableRawScanResultConsumer.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/AsyncMetaTableAccessor.MetaTableRawScanResultConsumer.html @@ -47,590 +47,589 @@ 039import org.apache.hadoop.hbase.MetaTableAccessor.QueryType; 040import org.apache.hadoop.hbase.MetaTableAccessor.Visitor; 041import org.apache.hadoop.hbase.classification.InterfaceAudience; -042import org.apache.hadoop.hbase.client.Connection; -043import org.apache.hadoop.hbase.client.Consistency; -044import org.apache.hadoop.hbase.client.Get; -045import org.apache.hadoop.hbase.client.RawAsyncTable; -046import org.apache.hadoop.hbase.client.RawScanResultConsumer; -047import org.apache.hadoop.hbase.client.RegionReplicaUtil; -048import org.apache.hadoop.hbase.client.Result; -049import org.apache.hadoop.hbase.client.Scan; -050import org.apache.hadoop.hbase.client.TableState; -051import org.apache.hadoop.hbase.client.Scan.ReadType; -052import org.apache.hadoop.hbase.exceptions.DeserializationException; -053import org.apache.hadoop.hbase.util.Bytes; -054import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -055import org.apache.hadoop.hbase.util.Pair; -056 -057/** -058 * The asynchronous meta table accessor. Used to read/write region and assignment information store -059 * in <code>hbase:meta</code>. -060 */ -061@InterfaceAudience.Private -062public class AsyncMetaTableAccessor { -063 -064 private static final Log LOG = LogFactory.getLog(AsyncMetaTableAccessor.class); +042import org.apache.hadoop.hbase.client.Consistency; +043import org.apache.hadoop.hbase.client.Get; +044import org.apache.hadoop.hbase.client.RawAsyncTable; +045import org.apache.hadoop.hbase.client.RawScanResultConsumer; +046import org.apache.hadoop.hbase.client.RegionReplicaUtil; +047import org.apache.hadoop.hbase.client.Result; +048import org.apache.hadoop.hbase.client.Scan; +049import org.apache.hadoop.hbase.client.TableState; +050import org.apache.hadoop.hbase.client.Scan.ReadType; +051import org.apache.hadoop.hbase.exceptions.DeserializationException; +052import org.apache.hadoop.hbase.util.Bytes; +053import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +054import org.apache.hadoop.hbase.util.Pair; +055 +056/** +057 * The asynchronous meta table accessor. Used to read/write region and assignment information store +058 * in <code>hbase:meta</code>. +059 */ +060@InterfaceAudience.Private +061public class AsyncMetaTableAccessor { +062 +063 private static final Log LOG = LogFactory.getLog(AsyncMetaTableAccessor.class); +064 065 -066 -067 /** The delimiter for meta columns for replicaIds &gt; 0 */ -068 private static final char META_REPLICA_ID_DELIMITER = '_'; -069 -070 /** A regex for parsing server columns from meta. See above javadoc for meta layout */ -071 private static final Pattern SERVER_COLUMN_PATTERN = Pattern -072 .compile("^server(_[0-9a-fA-F]{4})?$"); -073 -074 public static CompletableFuture<Boolean> tableExists(RawAsyncTable metaTable, TableName tableName) { -075 if (tableName.equals(META_TABLE_NAME)) { -076 return CompletableFuture.completedFuture(true); -077 } -078 return getTableState(metaTable, tableName).thenApply(Optional::isPresent); -079 } -080 -081 public static CompletableFuture<Optional<TableState>> getTableState(RawAsyncTable metaTable, -082 TableName tableName) { -083 CompletableFuture<Optional<TableState>> future = new CompletableFuture<>(); -084 Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getStateColumn()); -085 long time = EnvironmentEdgeManager.currentTime(); -086 try { -087 get.setTimeRange(0, time); -088 metaTable.get(get).whenComplete((result, error) -> { -089 if (error != null) { -090 future.completeExceptionally(error); -091 return; -092 } -093 try { -094 future.complete(getTableState(result)); -095 } catch (IOException e) { -096 future.completeExceptionally(e); -097 } -098 }); -099 } catch (IOException ioe) { -100 future.completeExceptionally(ioe); -101 } -102 return future; -103 } -104 -105 /** -106 * Returns the HRegionLocation from meta for the given region -107 * @param metaTable -108 * @param regionName region we're looking for -109 * @return HRegionLocation for the given region -110 */ -111 public static CompletableFuture<Optional<HRegionLocation>> getRegionLocation( -112 RawAsyncTable metaTable, byte[] regionName) { -113 CompletableFuture<Optional<HRegionLocation>> future = new CompletableFuture<>(); -114 try { -115 HRegionInfo parsedRegionInfo = MetaTableAccessor.parseRegionInfoFromRegionName(regionName); -116 metaTable.get( -117 new Get(MetaTableAccessor.getMetaKeyForRegion(parsedRegionInfo)) -118 .addFamily(HConstants.CATALOG_FAMILY)).whenComplete( -119 (r, err) -> { -120 if (err != null) { -121 future.completeExceptionally(err); -122 return; -123 } -124 future.complete(getRegionLocations(r).map( -125 locations -> locations.getRegionLocation(parsedRegionInfo.getReplicaId()))); -126 }); -127 } catch (IOException parseEx) { -128 LOG.warn("Failed to parse the passed region name: " + Bytes.toStringBinary(regionName)); -129 future.completeExceptionally(parseEx); -130 } -131 return future; -132 } -133 -134 /** -135 * Returns the HRegionLocation from meta for the given encoded region name -136 * @param metaTable -137 * @param encodedRegionName region we're looking for -138 * @return HRegionLocation for the given region -139 */ -140 public static CompletableFuture<Optional<HRegionLocation>> getRegionLocationWithEncodedName( -141 RawAsyncTable metaTable, byte[] encodedRegionName) { -142 CompletableFuture<Optional<HRegionLocation>> future = new CompletableFuture<>(); -143 metaTable.scanAll(new Scan().setReadType(ReadType.PREAD).addFamily(HConstants.CATALOG_FAMILY)) -144 .whenComplete( -145 (results, err) -> { -146 if (err != null) { -147 future.completeExceptionally(err); -148 return; -149 } -150 String encodedRegionNameStr = Bytes.toString(encodedRegionName); -151 results -152 .stream() -153 .filter(result -> !result.isEmpty()) -154 .filter(result -> MetaTableAccessor.getHRegionInfo(result) != null) -155 .forEach( -156 result -> { -157 getRegionLocations(result).ifPresent( -158 locations -> { -159 for (HRegionLocation location : locations.getRegionLocations()) { -160 if (location != null -161 && encodedRegionNameStr.equals(location.getRegionInfo() -162 .getEncodedName())) { -163 future.complete(Optional.of(location)); -164 return; -165 } -166 } -167 }); -168 }); -169 future.complete(Optional.empty()); -170 }); -171 return future; -172 } -173 -174 private static Optional<TableState> getTableState(Result r) throws IOException { -175 Cell cell = r.getColumnLatestCell(getTableFamily(), getStateColumn()); -176 if (cell == null) return Optional.empty(); -177 try { -178 return Optional.of(TableState.parseFrom( -179 TableName.valueOf(r.getRow()), -180 Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), cell.getValueOffset() -181 + cell.getValueLength()))); -182 } catch (DeserializationException e) { -183 throw new IOException("Failed to parse table state from result: " + r, e); -184 } -185 } -186 -187 /** -188 * Used to get all region locations for the specific table. -189 * @param metaTable -190 * @param tableName table we're looking for, can be null for getting all regions -191 * @return the list of region locations. The return value will be wrapped by a -192 * {@link CompletableFuture}. -193 */ -194 public static CompletableFuture<List<HRegionLocation>> getTableHRegionLocations( -195 RawAsyncTable metaTable, final Optional<TableName> tableName) { -196 CompletableFuture<List<HRegionLocation>> future = new CompletableFuture<>(); -197 getTableRegionsAndLocations(metaTable, tableName, true).whenComplete( -198 (locations, err) -> { -199 if (err != null) { -200 future.completeExceptionally(err); -201 } else if (locations == null || locations.isEmpty()) { -202 future.complete(Collections.emptyList()); -203 } else { -204 List<HRegionLocation> regionLocations = locations.stream() -205 .map(loc -> new HRegionLocation(loc.getFirst(), loc.getSecond())) -206 .collect(Collectors.toList()); -207 future.complete(regionLocations); -208 } -209 }); -210 return future; -211 } -212 -213 /** -214 * Used to get table regions' info and server. -215 * @param metaTable -216 * @param tableName table we're looking for, can be null for getting all regions -217 * @param excludeOfflinedSplitParents don't return split parents -218 * @return the list of regioninfos and server. The return value will be wrapped by a -219 * {@link CompletableFuture}. -220 */ -221 private static CompletableFuture<List<Pair<HRegionInfo, ServerName>>> getTableRegionsAndLocations( -222 RawAsyncTable metaTable, final Optional<TableName> tableName, -223 final boolean excludeOfflinedSplitParents) { -224 CompletableFuture<List<Pair<HRegionInfo, ServerName>>> future = new CompletableFuture<>(); -225 if (tableName.filter((t) -> t.equals(TableName.META_TABLE_NAME)).isPresent()) { -226 future.completeExceptionally(new IOException( -227 "This method can't be used to locate meta regions;" + " use MetaTableLocator instead")); -228 } -229 -230 // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress -231 CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor = new CollectingVisitor<Pair<HRegionInfo, ServerName>>() { -232 private Optional<RegionLocations> current = null; -233 -234 @Override -235 public boolean visit(Result r) throws IOException { -236 current = getRegionLocations(r); -237 if (!current.isPresent() || current.get().getRegionLocation().getRegionInfo() == null) { -238 LOG.warn("No serialized HRegionInfo in " + r); -239 return true; -240 } -241 HRegionInfo hri = current.get().getRegionLocation().getRegionInfo(); -242 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; -243 // Else call super and add this Result to the collection. -244 return super.visit(r); -245 } -246 -247 @Override -248 void add(Result r) { -249 if (!current.isPresent()) { -250 return; -251 } -252 for (HRegionLocation loc : current.get().getRegionLocations()) { -253 if (loc != null) { -254 this.results.add(new Pair<HRegionInfo, ServerName>(loc.getRegionInfo(), loc -255 .getServerName())); -256 } -257 } -258 } -259 }; -260 -261 scanMeta(metaTable, tableName, QueryType.REGION, visitor).whenComplete((v, error) -> { -262 if (error != null) { -263 future.completeExceptionally(error); -264 return; -265 } -266 future.complete(visitor.getResults()); -267 }); -268 return future; -269 } -270 -271 /** -272 * Performs a scan of META table for given table. -273 * @param metaTable -274 * @param tableName table withing we scan -275 * @param type scanned part of meta -276 * @param visitor Visitor invoked against each row -277 */ -278 private static CompletableFuture<Void> scanMeta(RawAsyncTable metaTable, -279 Optional<TableName> tableName, QueryType type, final Visitor visitor) { -280 return scanMeta(metaTable, getTableStartRowForMeta(tableName, type), -281 getTableStopRowForMeta(tableName, type), type, Integer.MAX_VALUE, visitor); -282 } -283 -284 /** -285 * Performs a scan of META table for given table. -286 * @param metaTable -287 * @param startRow Where to start the scan -288 * @param stopRow Where to stop the scan -289 * @param type scanned part of meta -290 * @param maxRows maximum rows to return -291 * @param visitor Visitor invoked against each row -292 */ -293 private static CompletableFuture<Void> scanMeta(RawAsyncTable metaTable, Optional<byte[]> startRow, -294 Optional<byte[]> stopRow, QueryType type, int maxRows, final Visitor visitor) { -295 int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; -296 Scan scan = getMetaScan(metaTable, rowUpperLimit); -297 for (byte[] family : type.getFamilies()) { -298 scan.addFamily(family); -299 } -300 startRow.ifPresent(scan::withStartRow); -301 stopRow.ifPresent(scan::withStopRow); -302 -303 if (LOG.isTraceEnabled()) { -304 LOG.trace("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) -305 + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" -306 + rowUpperLimit + " with caching=" + scan.getCaching()); -307 } -308 -309 CompletableFuture<Void> future = new CompletableFuture<Void>(); -310 metaTable.scan(scan, new MetaTableRawScanResultConsumer(rowUpperLimit, visitor, future)); -311 return future; -312 } -313 -314 private static final class MetaTableRawScanResultConsumer implements RawScanResultConsumer { -315 -316 private int currentRowCount; -317 -318 private final int rowUpperLimit; -319 -320 private final Visitor visitor; -321 -322 private final CompletableFuture<Void> future; -323 -324 MetaTableRawScanResultConsumer(int rowUpperLimit, Visitor visitor, CompletableFuture<Void> future) { -325 this.rowUpperLimit = rowUpperLimit; -326 this.visitor = visitor; -327 this.future = future; -328 this.currentRowCount = 0; -329 } -330 -331 @Override -332 public void onError(Throwable error) { -333 future.completeExceptionally(error); -334 } -335 -336 @Override -337 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NP_NONNULL_PARAM_VIOLATION", -338 justification = "https://github.com/findbugsproject/findbugs/issues/79") -339 public void onComplete() { -340 future.complete(null); -341 } -342 -343 @Override -344 public void onNext(Result[] results, ScanController controller) { -345 for (Result result : results) { -346 try { -347 if (!visitor.visit(result)) { -348 controller.terminate(); -349 } -350 } catch (IOException e) { -351 future.completeExceptionally(e); -352 controller.terminate(); -353 } -354 if (++currentRowCount >= rowUpperLimit) { -355 controller.terminate(); -356 } -357 } -358 } -359 } -360 -361 private static Scan getMetaScan(RawAsyncTable metaTable, int rowUpperLimit) { -362 Scan scan = new Scan(); -363 int scannerCaching = metaTable.getConfiguration().getInt(HConstants.HBASE_META_SCANNER_CACHING, -364 HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); -365 if (metaTable.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, -366 HConstants.DEFAULT_USE_META_REPLICAS)) { -367 scan.setConsistency(Consistency.TIMELINE); -368 } -369 if (rowUpperLimit <= scannerCaching) { -370 scan.setLimit(rowUpperLimit); -371 } -372 int rows = Math.min(rowUpperLimit, scannerCaching); -373 scan.setCaching(rows); -374 return scan; -375 } -376 -377 /** -378 * Returns an HRegionLocationList extracted from the result. -379 * @return an HRegionLocationList containing all locations for the region range or null if we -380 * can't deserialize the result. -381 */ -382 private static Optional<RegionLocations> getRegionLocations(final Result r) { -383 if (r == null) return Optional.empty(); -384 Optional<HRegionInfo> regionInfo = getHRegionInfo(r, getRegionInfoColumn()); -385 if (!regionInfo.isPresent()) return Optional.empty(); -386 -387 List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1); -388 NavigableMap<byte[], NavigableMap<byte[], byte[]>> familyMap = r.getNoVersionMap(); -389 -390 locations.add(getRegionLocation(r, regionInfo.get(), 0)); -391 -392 NavigableMap<byte[], byte[]> infoMap = familyMap.get(getCatalogFamily()); -393 if (infoMap == null) return Optional.of(new RegionLocations(locations)); -394 -395 // iterate until all serverName columns are seen -396 int replicaId = 0; -397 byte[] serverColumn = getServerColumn(replicaId); -398 SortedMap<byte[], byte[]> serverMap = null; -399 serverMap = infoMap.tailMap(serverColumn, false); -400 -401 if (serverMap.isEmpty()) return Optional.of(new RegionLocations(locations)); -402 -403 for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) { -404 replicaId = parseReplicaIdFromServerColumn(entry.getKey()); -405 if (replicaId < 0) { -406 break; -407 } -408 HRegionLocation location = getRegionLocation(r, regionInfo.get(), replicaId); -409 // In case the region replica is newly created, it's location might be null. We usually do not -410 // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. -411 if (location == null || location.getServerName() == null) { -412 locations.add(null); -413 } else { -414 locations.add(location); -415 } -416 } -417 -418 return Optional.of(new RegionLocations(locations)); -419 } -420 -421 /** -422 * Returns the HRegionLocation parsed from the given meta row Result -423 * for the given regionInfo and replicaId. The regionInfo can be the default region info -424 * for the replica. -425 * @param r the meta row result -426 * @param regionInfo RegionInfo for default replica -427 * @param replicaId the replicaId for the HRegionLocation -428 * @return HRegionLocation parsed from the given meta row Result for the given replicaId -429 */ -430 private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo, -431 final int replicaId) { -432 Optional<ServerName> serverName = getServerName(r, replicaId); -433 long seqNum = getSeqNumDuringOpen(r, replicaId); -434 HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId); -435 return new HRegionLocation(replicaInfo, serverName.orElse(null), seqNum); -436 } -437 -438 /** -439 * Returns a {@link ServerName} from catalog table {@link Result}. -440 * @param r Result to pull from -441 * @return A ServerName instance. -442 */ -443 private static Optional<ServerName> getServerName(final Result r, final int replicaId) { -444 byte[] serverColumn = getServerColumn(replicaId); -445 Cell cell = r.getColumnLatestCell(getCatalogFamily(), serverColumn); -446 if (cell == null || cell.getValueLength() == 0) return Optional.empty(); -447 String hostAndPort = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), -448 cell.getValueLength()); -449 byte[] startcodeColumn = getStartCodeColumn(replicaId); -450 cell = r.getColumnLatestCell(getCatalogFamily(), startcodeColumn); -451 if (cell == null || cell.getValueLength() == 0) return Optional.empty(); -452 try { -453 return Optional.of(ServerName.valueOf(hostAndPort, -454 Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()))); -455 } catch (IllegalArgumentException e) { -456 LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e); -457 return Optional.empty(); -458 } -459 } -460 -461 /** -462 * The latest seqnum that the server writing to meta observed when opening the region. -463 * E.g. the seqNum when the result of {@link #getServerName(Result, int)} was written. -464 * @param r Result to pull the seqNum from -465 * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written. -466 */ -467 private static long getSeqNumDuringOpen(final Result r, final int replicaId) { -468 Cell cell = r.getColumnLatestCell(getCatalogFamily(), getSeqNumColumn(replicaId)); -469 if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM; -470 return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); -471 } -472 -473 /** -474 * @param tableName table we're working with -475 * @return start row for scanning META according to query type -476 */ -477 private static Optional<byte[]> getTableStartRowForMeta(Optional<TableName> tableName, -478 QueryType type) { -479 return tableName.map((table) -> { -480 switch (type) { -481 case REGION: -482 byte[] startRow = new byte[table.getName().length + 2]; -483 System.arraycopy(table.getName(), 0, startRow, 0, table.getName().length); -484 startRow[startRow.length - 2] = HConstants.DELIMITER; -485 startRow[startRow.length - 1] = HConstants.DELIMITER; -486 return startRow; -487 case ALL: -488 case TABLE: -489 default: -490 return table.getName(); -491 } -492 }); -493 } -494 -495 /** -496 * @param tableName table we're working with -497 * @return stop row for scanning META according to query type -498 */ -499 private static Optional<byte[]> getTableStopRowForMeta(Optional<TableName> tableName, -500 QueryType type) { -501 return tableName.map((table) -> { -502 final byte[] stopRow; -503 switch (type) { -504 case REGION: -505 stopRow = new byte[table.getName().length + 3]; -506 System.arraycopy(table.getName(), 0, stopRow, 0, table.getName().length); -507 stopRow[stopRow.length - 3] = ' '; -508 stopRow[stopRow.length - 2] = HConstants.DELIMITER; -509 stopRow[stopRow.length - 1] = HConstants.DELIMITER; -510 break; -511 case ALL: -512 case TABLE: -513 default: -514 stopRow = new byte[table.getName().length + 1]; -515 System.arraycopy(table.getName(), 0, stopRow, 0, table.getName().length); -516 stopRow[stopRow.length - 1] = ' '; -517 break; -518 } -519 return stopRow; -520 }); -521 } -522 -523 /** -524 * Returns the HRegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and -525 * <code>qualifier</code> of the catalog table result. -526 * @param r a Result object from the catalog table scan -527 * @param qualifier Column family qualifier -528 * @return An HRegionInfo instance. -529 */ -530 private static Optional<HRegionInfo> getHRegionInfo(final Result r, byte[] qualifier) { -531 Cell cell = r.getColumnLatestCell(getCatalogFamily(), qualifier); -532 if (cell == null) return Optional.empty(); -533 return Optional.ofNullable(HRegionInfo.parseFromOrNull(cell.getValueArray(), -534 cell.getValueOffset(), cell.getValueLength())); -535 } -536 -537 /** -538 * Returns the column family used for meta columns. -539 * @return HConstants.CATALOG_FAMILY. -540 */ -541 private static byte[] getCatalogFamily() { -542 return HConstants.CATALOG_FAMILY; -543 } -544 -545 /** -546 * Returns the column family used for table columns. -547 * @return HConstants.TABLE_FAMILY. -548 */ -549 private static byte[] getTableFamily() { -550 return HConstants.TABLE_FAMILY; -551 } -552 -553 /** -554 * Returns the column qualifier for serialized region info -555 * @return HConstants.REGIONINFO_QUALIFIER -556 */ -557 private static byte[] getRegionInfoColumn() { -558 return HConstants.REGIONINFO_QUALIFIER; -559 } -560 -561 /** -562 * Returns the column qualifier for serialized table state -563 * @return HConstants.TABLE_STATE_QUALIFIER -564 */ -565 private static byte[] getStateColumn() { -566 return HConstants.TABLE_STATE_QUALIFIER; -567 } -568 -569 /** -570 * Returns the column qualifier for server column for replicaId -571 * @param replicaId the replicaId of the region -572 * @return a byte[] for server column qualifier -573 */ -574 private static byte[] getServerColumn(int replicaId) { -575 return replicaId == 0 -576 ? HConstants.SERVER_QUALIFIER -577 : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER -578 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId)); -579 } -580 -581 /** -582 * Returns the column qualifier for server start code column for replicaId -583 * @param replicaId the replicaId of the region -584 * @return a byte[] for server start code column qualifier -585 */ -586 private static byte[] getStartCodeColumn(int replicaId) { -587 return replicaId == 0 -588 ? HConstants.STARTCODE_QUALIFIER -589 : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER -590 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId)); -591 } -592 -593 /** -594 * Returns the column qualifier for seqNum column for replicaId -595 * @param replicaId the replicaId of the region -596 * @return a byte[] for seqNum column qualifier -597 */ -598 private static byte[] getSeqNumColumn(int replicaId) { -599 return replicaId == 0 -600 ? HConstants.SEQNUM_QUALIFIER -601 : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER -602 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId)); -603 } -604 -605 /** -606 * Parses the replicaId from the server column qualifier. See top of the class javadoc -607 * for the actual meta layout -608 * @param serverColumn the column qualifier -609 * @return an int for the replicaId -610 */ -611 private static int parseReplicaIdFromServerColumn(byte[] serverColumn) { -612 String serverStr = Bytes.toString(serverColumn); -613 -614 Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr); -615 if (matcher.matches() && matcher.groupCount() > 0) { -616 String group = matcher.group(1); -617 if (group != null && group.length() > 0) { -618 return Integer.parseInt(group.substring(1), 16); -619 } else { -620 return 0; -621 } -622 } -623 return -1; -624 } -625} +066 /** The delimiter for meta columns for replicaIds &gt; 0 */ +067 private static final char META_REPLICA_ID_DELIMITER = '_'; +068 +069 /** A regex for parsing server columns from meta. See above javadoc for meta layout */ +070 private static final Pattern SERVER_COLUMN_PATTERN = Pattern +071 .compile("^server(_[0-9a-fA-F]{4})?$"); +072 +073 public static CompletableFuture<Boolean> tableExists(RawAsyncTable metaTable, TableName tableName) { +074 if (tableName.equals(META_TABLE_NAME)) { +075 return CompletableFuture.completedFuture(true); +076 } +077 return getTableState(metaTable, tableName).thenApply(Optional::isPresent); +078 } +079 +080 public static CompletableFuture<Optional<TableState>> getTableState(RawAsyncTable metaTable, +081 TableName tableName) { +082 CompletableFuture<Optional<TableState>> future = new CompletableFuture<>(); +083 Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getStateColumn()); +084 long time = EnvironmentEdgeManager.currentTime(); +085 try { +086 get.setTimeRange(0, time); +087 metaTable.get(get).whenComplete((result, error) -> { +088 if (error != null) { +089 future.completeExceptionally(error); +090 return; +091 } +092 try { +093 future.complete(getTableState(result)); +094 } catch (IOException e) { +095 future.completeExceptionally(e); +096 } +097 }); +098 } catch (IOException ioe) { +099 future.completeExceptionally(ioe); +100 } +101 return future; +102 } +103 +104 /** +105 * Returns the HRegionLocation from meta for the given region +106 * @param metaTable +107 * @param regionName region we're looking for +108 * @return HRegionLocation for the given region +109 */ +110 public static CompletableFuture<Optional<HRegionLocation>> getRegionLocation( +111 RawAsyncTable metaTable, byte[] regionName) { +112 CompletableFuture<Optional<HRegionLocation>> future = new CompletableFuture<>(); +113 try { +114 HRegionInfo parsedRegionInfo = MetaTableAccessor.parseRegionInfoFromRegionName(regionName); +115 metaTable.get( +116 new Get(MetaTableAccessor.getMetaKeyForRegion(parsedRegionInfo)) +117 .addFamily(HConstants.CATALOG_FAMILY)).whenComplete( +118 (r, err) -> { +119 if (err != null) { +120 future.completeExceptionally(err); +121 return; +122 } +123 future.complete(getRegionLocations(r).map( +124 locations -> locations.getRegionLocation(parsedRegionInfo.getReplicaId()))); +125 }); +126 } catch (IOException parseEx) { +127 LOG.warn("Failed to parse the passed region name: " + Bytes.toStringBinary(regionName)); +128 future.completeExceptionally(parseEx); +129 } +130 return future; +131 } +132 +133 /** +134 * Returns the HRegionLocation from meta for the given encoded region name +135 * @param metaTable +136 * @param encodedRegionName region we're looking for +137 * @return HRegionLocation for the given region +138 */ +139 public static CompletableFuture<Optional<HRegionLocation>> getRegionLocationWithEncodedName( +140 RawAsyncTable metaTable, byte[] encodedRegionName) { +141 CompletableFuture<Optional<HRegionLocation>> future = new CompletableFuture<>(); +142 metaTable.scanAll(new Scan().setReadType(ReadType.PREAD).addFamily(HConstants.CATALOG_FAMILY)) +143 .whenComplete( +144 (results, err) -> { +145 if (err != null) { +146 future.completeExceptionally(err); +147 return; +148 } +149 String encodedRegionNameStr = Bytes.toString(encodedRegionName); +150 results +151 .stream() +152 .filter(result -> !result.isEmpty()) +153 .filter(result -> MetaTableAccessor.getHRegionInfo(result) != null) +154 .forEach( +155 result -> { +156 getRegionLocations(result).ifPresent( +157 locations -> { +158 for (HRegionLocation location : locations.getRegionLocations()) { +159 if (location != null +160 && encodedRegionNameStr.equals(location.getRegionInfo() +161 .getEncodedName())) { +162 future.complete(Optional.of(location)); +163 return; +164 } +165 } +166 }); +167 }); +168 future.complete(Optional.empty()); +169 }); +170 return future; +171 } +172 +173 private static Optional<TableState> getTableState(Result r) throws IOException { +174 Cell cell = r.getColumnLatestCell(getTableFamily(), getStateColumn()); +175 if (cell == null) return Optional.empty(); +176 try { +177 return Optional.of(TableState.parseFrom( +178 TableName.valueOf(r.getRow()), +179 Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), cell.getValueOffset() +180 + cell.getValueLength()))); +181 } catch (DeserializationException e) { +182 throw new IOException("Failed to parse table state from result: " + r, e); +183 } +184 } +185 +186 /** +187 * Used to get all region locations for the specific table. +188 * @param metaTable +189 * @param tableName table we're looking for, can be null for getting all regions +190 * @return the list of region locations. The return value will be wrapped by a +191 * {@link CompletableFuture}. +192 */ +193 public static CompletableFuture<List<HRegionLocation>> getTableHRegionLocations( +194 RawAsyncTable metaTable, final Optional<TableName> tableName) { +195 CompletableFuture<List<HRegionLocation>> future = new CompletableFuture<>(); +196 getTableRegionsAndLocations(metaTable, tableName, true).whenComplete( +197 (locations, err) -> { +198 if (err != null) { +199 future.completeExceptionally(err); +200 } else if (locations == null || locations.isEmpty()) { +201 future.complete(Collections.emptyList()); +202 } else { +203 List<HRegionLocation> regionLocations = locations.stream() +204 .map(loc -> new HRegionLocation(loc.getFirst(), loc.getSecond())) +205 .collect(Collectors.toList()); +206 future.complete(regionLocations); +207 } +208 }); +209 return future; +210 } +211 +212 /** +213 * Used to get table regions' info and server. +214 * @param metaTable +215 * @param tableName table we're looking for, can be null for getting all regions +216 * @param excludeOfflinedSplitParents don't return split parents +217 * @return the list of regioninfos and server. The return value will be wrapped by a +218 * {@link CompletableFuture}. +219 */ +220 private static CompletableFuture<List<Pair<HRegionInfo, ServerName>>> getTableRegionsAndLocations( +221 RawAsyncTable metaTable, final Optional<TableName> tableName, +222 final boolean excludeOfflinedSplitParents) { +223 CompletableFuture<List<Pair<HRegionInfo, ServerName>>> future = new CompletableFuture<>(); +224 if (tableName.filter((t) -> t.equals(TableName.META_TABLE_NAME)).isPresent()) { +225 future.completeExceptionally(new IOException( +226 "This method can't be used to locate meta regions;" + " use MetaTableLocator instead")); +227 } +228 +229 // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress +230 CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor = new CollectingVisitor<Pair<HRegionInfo, ServerName>>() { +231 private Optional<RegionLocations> current = null; +232 +233 @Override +234 public boolean visit(Result r) throws IOException { +235 current = getRegionLocations(r); +236 if (!current.isPresent() || current.get().getRegionLocation().getRegionInfo() == null) { +237 LOG.warn("No serialized HRegionInfo in " + r); +238 return true; +239 } +240 HRegionInfo hri = current.get().getRegionLocation().getRegionInfo(); +241 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; +242 // Else call super and add this Result to the collection. +243 return super.visit(r); +244 } +245 +246 @Override +247 void add(Result r) { +248 if (!current.isPresent()) { +249 return; +250 } +251 for (HRegionLocation loc : current.get().getRegionLocations()) { +252 if (loc != null) { +253 this.results.add(new Pair<HRegionInfo, ServerName>(loc.getRegionInfo(), loc +254 .getServerName())); +255 } +256 } +257 } +258 }; +259 +260 scanMeta(metaTable, tableName, QueryType.REGION, visitor).whenComplete((v, error) -> { +261 if (error != null) { +262 future.completeExceptionally(error); +263 return; +264 } +265 future.complete(visitor.getResults()); +266 }); +267 return future; +268 } +269 +270 /** +271 * Performs a scan of META table for given table. +272 * @param metaTable +273 * @param tableName table withing we scan +274 * @param type scanned part of meta +275 * @param visitor Visitor invoked against each row +276 */ +277 private static CompletableFuture<Void> scanMeta(RawAsyncTable metaTable, +278 Optional<TableName> tableName, QueryType type, final Visitor visitor) { +279 return scanMeta(metaTable, getTableStartRowForMeta(tableName, type), +280 getTableStopRowForMeta(tableName, type), type, Integer.MAX_VALUE, visitor); +281 } +282 +283 /** +284 * Performs a scan of META table for given table. +285 * @param metaTable +286 * @param startRow Where to start the scan +287 * @param stopRow Where to stop the scan +288 * @param type scanned part of meta +289 * @param maxRows maximum rows to return +290 * @param visitor Visitor invoked against each row +291 */ +292 private static CompletableFuture<Void> scanMeta(RawAsyncTable metaTable, Optional<byte[]> startRow, +293 Optional<byte[]> stopRow, QueryType type, int maxRows, final Visitor visitor) { +294 int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; +295 Scan scan = getMetaScan(metaTable, rowUpperLimit); +296 for (byte[] family : type.getFamilies()) { +297 scan.addFamily(family); +298 } +299 startRow.ifPresent(scan::withStartRow); +300 stopRow.ifPresent(scan::withStopRow); +301 +302 if (LOG.isTraceEnabled()) { +303 LOG.trace("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) +304 + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" +305 + rowUpperLimit + " with caching=" + scan.getCaching()); +306 } +307 +308 CompletableFuture<Void> future = new CompletableFuture<Void>(); +309 metaTable.scan(scan, new MetaTableRawScanResultConsumer(rowUpperLimit, visitor, future)); +310 return future; +311 } +312 +313 private static final class MetaTableRawScanResultConsumer implements RawScanResultConsumer { +314 +315 private int currentRowCount; +316 +317 private final int rowUpperLimit; +318 +319 private final Visitor visitor; +320 +321 private final CompletableFuture<Void> future; +322 +323 MetaTableRawScanResultConsumer(int rowUpperLimit, Visitor visitor, CompletableFuture<Void> future) { +324 this.rowUpperLimit = rowUpperLimit; +325 this.visitor = visitor; +326 this.future = future; +327 this.currentRowCount = 0; +328 } +329 +330 @Override +