Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0DA13200D60 for ; Fri, 10 Nov 2017 16:29:57 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 0C1C0160C04; Fri, 10 Nov 2017 15:29:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 71A7C160C06 for ; Fri, 10 Nov 2017 16:29:55 +0100 (CET) Received: (qmail 43137 invoked by uid 500); 10 Nov 2017 15:29:51 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 41410 invoked by uid 99); 10 Nov 2017 15:29:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Nov 2017 15:29:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B3974F5DC1; Fri, 10 Nov 2017 15:29:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Fri, 10 Nov 2017 15:30:13 -0000 Message-Id: <23bb7ad798414328b9488e1003a25aad@git.apache.org> In-Reply-To: <6d878f2976364f088768a3569569c0a0@git.apache.org> References: <6d878f2976364f088768a3569569c0a0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [26/51] [partial] hbase-site git commit: Published site at . archived-at: Fri, 10 Nov 2017 15:29:57 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a108018f/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/RegionStateStore.RegionStateVisitor.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/RegionStateStore.RegionStateVisitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/RegionStateStore.RegionStateVisitor.html index 845f5e6..84e9e52 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/RegionStateStore.RegionStateVisitor.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/RegionStateStore.RegionStateVisitor.html @@ -98,241 +98,248 @@ 090 @Override 091 public boolean visit(final Result r) throws IOException { 092 if (r != null && !r.isEmpty()) { -093 long st = System.currentTimeMillis(); -094 visitMetaEntry(visitor, r); -095 long et = System.currentTimeMillis(); -096 LOG.info("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st)); -097 } else if (isDebugEnabled) { -098 LOG.debug("NULL result from meta - ignoring but this is strange."); -099 } -100 return true; -101 } -102 }); -103 } -104 -105 private void visitMetaEntry(final RegionStateVisitor visitor, final Result result) -106 throws IOException { -107 final RegionLocations rl = MetaTableAccessor.getRegionLocations(result); -108 if (rl == null) return; +093 long st = 0; +094 if (LOG.isTraceEnabled()) { +095 st = System.currentTimeMillis(); +096 } +097 visitMetaEntry(visitor, r); +098 if (LOG.isTraceEnabled()) { +099 long et = System.currentTimeMillis(); +100 LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st)); +101 } +102 } else if (isDebugEnabled) { +103 LOG.debug("NULL result from meta - ignoring but this is strange."); +104 } +105 return true; +106 } +107 }); +108 } 109 -110 final HRegionLocation[] locations = rl.getRegionLocations(); -111 if (locations == null) return; -112 -113 for (int i = 0; i < locations.length; ++i) { -114 final HRegionLocation hrl = locations[i]; -115 if (hrl == null) continue; -116 -117 final RegionInfo regionInfo = hrl.getRegionInfo(); -118 if (regionInfo == null) continue; -119 -120 final int replicaId = regionInfo.getReplicaId(); -121 final State state = getRegionState(result, replicaId); -122 -123 final ServerName lastHost = hrl.getServerName(); -124 final ServerName regionLocation = getRegionServer(result, replicaId); -125 final long openSeqNum = -1; -126 -127 // TODO: move under trace, now is visible for debugging -128 LOG.info(String.format("Load hbase:meta entry region=%s regionState=%s lastHost=%s regionLocation=%s", -129 regionInfo, state, lastHost, regionLocation)); -130 -131 visitor.visitRegionState(regionInfo, state, regionLocation, lastHost, openSeqNum); -132 } -133 } -134 -135 public void updateRegionLocation(final RegionInfo regionInfo, final State state, -136 final ServerName regionLocation, final ServerName lastHost, final long openSeqNum, -137 final long pid) -138 throws IOException { -139 if (regionInfo.isMetaRegion()) { -140 updateMetaLocation(regionInfo, regionLocation); -141 } else { -142 updateUserRegionLocation(regionInfo, state, regionLocation, lastHost, openSeqNum, pid); -143 } -144 } -145 -146 public void updateRegionState(final long openSeqNum, final long pid, -147 final RegionState newState, final RegionState oldState) throws IOException { -148 updateRegionLocation(newState.getRegion(), newState.getState(), newState.getServerName(), -149 oldState != null ? oldState.getServerName() : null, openSeqNum, pid); -150 } -151 -152 protected void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName) -153 throws IOException { -154 try { -155 MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, -156 regionInfo.getReplicaId(), State.OPEN); -157 } catch (KeeperException e) { -158 throw new IOException(e); -159 } -160 } -161 -162 protected void updateUserRegionLocation(final RegionInfo regionInfo, final State state, -163 final ServerName regionLocation, final ServerName lastHost, final long openSeqNum, -164 final long pid) -165 throws IOException { -166 final int replicaId = regionInfo.getReplicaId(); -167 final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo)); -168 MetaTableAccessor.addRegionInfo(put, regionInfo); -169 final StringBuilder info = new StringBuilder("pid=" + pid + " updating hbase:meta row="); -170 info.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state); -171 if (openSeqNum >= 0) { -172 Preconditions.checkArgument(state == State.OPEN && regionLocation != null, -173 "Open region should be on a server"); -174 MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, -1, replicaId); -175 info.append(", openSeqNum=").append(openSeqNum); -176 info.append(", regionLocation=").append(regionLocation); -177 } else if (regionLocation != null && !regionLocation.equals(lastHost)) { -178 // Ideally, if no regionLocation, write null to the hbase:meta but this will confuse clients -179 // currently; they want a server to hit. TODO: Make clients wait if no location. -180 put.addImmutable(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId), -181 Bytes.toBytes(regionLocation.getServerName())); -182 info.append(", regionLocation=").append(regionLocation); -183 } -184 put.addImmutable(HConstants.CATALOG_FAMILY, getStateColumn(replicaId), -185 Bytes.toBytes(state.name())); -186 LOG.info(info); -187 -188 final boolean serialReplication = hasSerialReplicationScope(regionInfo.getTable()); -189 if (serialReplication && state == State.OPEN) { -190 Put barrierPut = MetaTableAccessor.makeBarrierPut(regionInfo.getEncodedNameAsBytes(), -191 openSeqNum, regionInfo.getTable().getName()); -192 updateRegionLocation(regionInfo, state, put, barrierPut); -193 } else { -194 updateRegionLocation(regionInfo, state, put); -195 } -196 } -197 -198 protected void updateRegionLocation(final RegionInfo regionInfo, final State state, -199 final Put... put) throws IOException { -200 synchronized (this) { -201 if (multiHConnection == null) { -202 multiHConnection = new MultiHConnection(master.getConfiguration(), 1); -203 } -204 } -205 -206 try { -207 multiHConnection.processBatchCallback(Arrays.asList(put), TableName.META_TABLE_NAME, null, null); -208 } catch (IOException e) { -209 // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! -210 // In tests we abort the Master! -211 String msg = String.format("FAILED persisting region=%s state=%s", -212 regionInfo.getShortNameToLog(), state); -213 LOG.error(msg, e); -214 master.abort(msg, e); -215 throw e; -216 } -217 } -218 -219 // ============================================================================================ -220 // Update Region Splitting State helpers -221 // ============================================================================================ -222 public void splitRegion(final RegionInfo parent, final RegionInfo hriA, -223 final RegionInfo hriB, final ServerName serverName) throws IOException { -224 final TableDescriptor htd = getTableDescriptor(parent.getTable()); -225 MetaTableAccessor.splitRegion(master.getConnection(), parent, hriA, hriB, serverName, -226 getRegionReplication(htd), hasSerialReplicationScope(htd)); -227 } -228 -229 // ============================================================================================ -230 // Update Region Merging State helpers -231 // ============================================================================================ -232 public void mergeRegions(final RegionInfo parent, final RegionInfo hriA, -233 final RegionInfo hriB, final ServerName serverName) throws IOException { -234 final TableDescriptor htd = getTableDescriptor(parent.getTable()); -235 MetaTableAccessor.mergeRegions(master.getConnection(), parent, hriA, hriB, serverName, -236 getRegionReplication(htd), EnvironmentEdgeManager.currentTime(), -237 hasSerialReplicationScope(htd)); -238 } -239 -240 // ============================================================================================ -241 // Delete Region State helpers -242 // ============================================================================================ -243 public void deleteRegion(final RegionInfo regionInfo) throws IOException { -244 deleteRegions(Collections.singletonList(regionInfo)); -245 } -246 -247 public void deleteRegions(final List<RegionInfo> regions) throws IOException { -248 MetaTableAccessor.deleteRegions(master.getConnection(), regions); -249 } -250 -251 // ========================================================================== -252 // Table Descriptors helpers -253 // ========================================================================== -254 private boolean hasSerialReplicationScope(final TableName tableName) throws IOException { -255 return hasSerialReplicationScope(getTableDescriptor(tableName)); -256 } -257 -258 private boolean hasSerialReplicationScope(final TableDescriptor htd) { -259 return (htd != null)? htd.hasSerialReplicationScope(): false; -260 } -261 -262 private int getRegionReplication(final TableDescriptor htd) { -263 return (htd != null) ? htd.getRegionReplication() : 1; -264 } -265 -266 private TableDescriptor getTableDescriptor(final TableName tableName) throws IOException { -267 return master.getTableDescriptors().get(tableName); -268 } -269 -270 // ========================================================================== -271 // Server Name -272 // ========================================================================== -273 -274 /** -275 * Returns the {@link ServerName} from catalog table {@link Result} -276 * where the region is transitioning. It should be the same as -277 * {@link MetaTableAccessor#getServerName(Result,int)} if the server is at OPEN state. -278 * @param r Result to pull the transitioning server name from -279 * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)} -280 * if necessary fields not found or empty. -281 */ -282 static ServerName getRegionServer(final Result r, int replicaId) { -283 final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, -284 getServerNameColumn(replicaId)); -285 if (cell == null || cell.getValueLength() == 0) { -286 RegionLocations locations = MetaTableAccessor.getRegionLocations(r); -287 if (locations != null) { -288 HRegionLocation location = locations.getRegionLocation(replicaId); -289 if (location != null) { -290 return location.getServerName(); -291 } -292 } -293 return null; -294 } -295 return ServerName.parseServerName(Bytes.toString(cell.getValueArray(), -296 cell.getValueOffset(), cell.getValueLength())); -297 } -298 -299 private static byte[] getServerNameColumn(int replicaId) { -300 return replicaId == 0 -301 ? HConstants.SERVERNAME_QUALIFIER -302 : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER -303 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); -304 } -305 -306 // ========================================================================== -307 // Region State -308 // ========================================================================== -309 -310 /** -311 * Pull the region state from a catalog table {@link Result}. -312 * @param r Result to pull the region state from -313 * @return the region state, or OPEN if there's no value written. -314 */ -315 protected State getRegionState(final Result r, int replicaId) { -316 Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getStateColumn(replicaId)); -317 if (cell == null || cell.getValueLength() == 0) return State.OPENING; -318 return State.valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); -319 } -320 -321 private static byte[] getStateColumn(int replicaId) { -322 return replicaId == 0 -323 ? HConstants.STATE_QUALIFIER -324 : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER -325 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); +110 private void visitMetaEntry(final RegionStateVisitor visitor, final Result result) +111 throws IOException { +112 final RegionLocations rl = MetaTableAccessor.getRegionLocations(result); +113 if (rl == null) return; +114 +115 final HRegionLocation[] locations = rl.getRegionLocations(); +116 if (locations == null) return; +117 +118 for (int i = 0; i < locations.length; ++i) { +119 final HRegionLocation hrl = locations[i]; +120 if (hrl == null) continue; +121 +122 final RegionInfo regionInfo = hrl.getRegionInfo(); +123 if (regionInfo == null) continue; +124 +125 final int replicaId = regionInfo.getReplicaId(); +126 final State state = getRegionState(result, replicaId); +127 +128 final ServerName lastHost = hrl.getServerName(); +129 final ServerName regionLocation = getRegionServer(result, replicaId); +130 final long openSeqNum = -1; +131 +132 // TODO: move under trace, now is visible for debugging +133 LOG.info(String.format("Load hbase:meta entry region=%s regionState=%s lastHost=%s regionLocation=%s", +134 regionInfo, state, lastHost, regionLocation)); +135 +136 visitor.visitRegionState(regionInfo, state, regionLocation, lastHost, openSeqNum); +137 } +138 } +139 +140 public void updateRegionLocation(final RegionInfo regionInfo, final State state, +141 final ServerName regionLocation, final ServerName lastHost, final long openSeqNum, +142 final long pid) +143 throws IOException { +144 if (regionInfo.isMetaRegion()) { +145 updateMetaLocation(regionInfo, regionLocation); +146 } else { +147 updateUserRegionLocation(regionInfo, state, regionLocation, lastHost, openSeqNum, pid); +148 } +149 } +150 +151 public void updateRegionState(final long openSeqNum, final long pid, +152 final RegionState newState, final RegionState oldState) throws IOException { +153 updateRegionLocation(newState.getRegion(), newState.getState(), newState.getServerName(), +154 oldState != null ? oldState.getServerName() : null, openSeqNum, pid); +155 } +156 +157 protected void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName) +158 throws IOException { +159 try { +160 MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, +161 regionInfo.getReplicaId(), State.OPEN); +162 } catch (KeeperException e) { +163 throw new IOException(e); +164 } +165 } +166 +167 protected void updateUserRegionLocation(final RegionInfo regionInfo, final State state, +168 final ServerName regionLocation, final ServerName lastHost, final long openSeqNum, +169 final long pid) +170 throws IOException { +171 final int replicaId = regionInfo.getReplicaId(); +172 final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo)); +173 MetaTableAccessor.addRegionInfo(put, regionInfo); +174 final StringBuilder info = new StringBuilder("pid=" + pid + " updating hbase:meta row="); +175 info.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state); +176 if (openSeqNum >= 0) { +177 Preconditions.checkArgument(state == State.OPEN && regionLocation != null, +178 "Open region should be on a server"); +179 MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, -1, replicaId); +180 info.append(", openSeqNum=").append(openSeqNum); +181 info.append(", regionLocation=").append(regionLocation); +182 } else if (regionLocation != null && !regionLocation.equals(lastHost)) { +183 // Ideally, if no regionLocation, write null to the hbase:meta but this will confuse clients +184 // currently; they want a server to hit. TODO: Make clients wait if no location. +185 put.addImmutable(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId), +186 Bytes.toBytes(regionLocation.getServerName())); +187 info.append(", regionLocation=").append(regionLocation); +188 } +189 put.addImmutable(HConstants.CATALOG_FAMILY, getStateColumn(replicaId), +190 Bytes.toBytes(state.name())); +191 LOG.info(info); +192 +193 final boolean serialReplication = hasSerialReplicationScope(regionInfo.getTable()); +194 if (serialReplication && state == State.OPEN) { +195 Put barrierPut = MetaTableAccessor.makeBarrierPut(regionInfo.getEncodedNameAsBytes(), +196 openSeqNum, regionInfo.getTable().getName()); +197 updateRegionLocation(regionInfo, state, put, barrierPut); +198 } else { +199 updateRegionLocation(regionInfo, state, put); +200 } +201 } +202 +203 protected void updateRegionLocation(final RegionInfo regionInfo, final State state, +204 final Put... put) throws IOException { +205 synchronized (this) { +206 if (multiHConnection == null) { +207 multiHConnection = new MultiHConnection(master.getConfiguration(), 1); +208 } +209 } +210 +211 try { +212 multiHConnection.processBatchCallback(Arrays.asList(put), TableName.META_TABLE_NAME, null, null); +213 } catch (IOException e) { +214 // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! +215 // In tests we abort the Master! +216 String msg = String.format("FAILED persisting region=%s state=%s", +217 regionInfo.getShortNameToLog(), state); +218 LOG.error(msg, e); +219 master.abort(msg, e); +220 throw e; +221 } +222 } +223 +224 // ============================================================================================ +225 // Update Region Splitting State helpers +226 // ============================================================================================ +227 public void splitRegion(final RegionInfo parent, final RegionInfo hriA, +228 final RegionInfo hriB, final ServerName serverName) throws IOException { +229 final TableDescriptor htd = getTableDescriptor(parent.getTable()); +230 MetaTableAccessor.splitRegion(master.getConnection(), parent, hriA, hriB, serverName, +231 getRegionReplication(htd), hasSerialReplicationScope(htd)); +232 } +233 +234 // ============================================================================================ +235 // Update Region Merging State helpers +236 // ============================================================================================ +237 public void mergeRegions(final RegionInfo parent, final RegionInfo hriA, +238 final RegionInfo hriB, final ServerName serverName) throws IOException { +239 final TableDescriptor htd = getTableDescriptor(parent.getTable()); +240 MetaTableAccessor.mergeRegions(master.getConnection(), parent, hriA, hriB, serverName, +241 getRegionReplication(htd), EnvironmentEdgeManager.currentTime(), +242 hasSerialReplicationScope(htd)); +243 } +244 +245 // ============================================================================================ +246 // Delete Region State helpers +247 // ============================================================================================ +248 public void deleteRegion(final RegionInfo regionInfo) throws IOException { +249 deleteRegions(Collections.singletonList(regionInfo)); +250 } +251 +252 public void deleteRegions(final List<RegionInfo> regions) throws IOException { +253 MetaTableAccessor.deleteRegions(master.getConnection(), regions); +254 } +255 +256 // ========================================================================== +257 // Table Descriptors helpers +258 // ========================================================================== +259 private boolean hasSerialReplicationScope(final TableName tableName) throws IOException { +260 return hasSerialReplicationScope(getTableDescriptor(tableName)); +261 } +262 +263 private boolean hasSerialReplicationScope(final TableDescriptor htd) { +264 return (htd != null)? htd.hasSerialReplicationScope(): false; +265 } +266 +267 private int getRegionReplication(final TableDescriptor htd) { +268 return (htd != null) ? htd.getRegionReplication() : 1; +269 } +270 +271 private TableDescriptor getTableDescriptor(final TableName tableName) throws IOException { +272 return master.getTableDescriptors().get(tableName); +273 } +274 +275 // ========================================================================== +276 // Server Name +277 // ========================================================================== +278 +279 /** +280 * Returns the {@link ServerName} from catalog table {@link Result} +281 * where the region is transitioning. It should be the same as +282 * {@link MetaTableAccessor#getServerName(Result,int)} if the server is at OPEN state. +283 * @param r Result to pull the transitioning server name from +284 * @return A ServerName instance or {@link MetaTableAccessor#getServerName(Result,int)} +285 * if necessary fields not found or empty. +286 */ +287 static ServerName getRegionServer(final Result r, int replicaId) { +288 final Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, +289 getServerNameColumn(replicaId)); +290 if (cell == null || cell.getValueLength() == 0) { +291 RegionLocations locations = MetaTableAccessor.getRegionLocations(r); +292 if (locations != null) { +293 HRegionLocation location = locations.getRegionLocation(replicaId); +294 if (location != null) { +295 return location.getServerName(); +296 } +297 } +298 return null; +299 } +300 return ServerName.parseServerName(Bytes.toString(cell.getValueArray(), +301 cell.getValueOffset(), cell.getValueLength())); +302 } +303 +304 private static byte[] getServerNameColumn(int replicaId) { +305 return replicaId == 0 +306 ? HConstants.SERVERNAME_QUALIFIER +307 : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER +308 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); +309 } +310 +311 // ========================================================================== +312 // Region State +313 // ========================================================================== +314 +315 /** +316 * Pull the region state from a catalog table {@link Result}. +317 * @param r Result to pull the region state from +318 * @return the region state, or null if unknown. +319 */ +320 protected State getRegionState(final Result r, int replicaId) { +321 Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getStateColumn(replicaId)); +322 if (cell == null || cell.getValueLength() == 0) { +323 return null; +324 } +325 return State.valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); 326 } -327} +327 +328 private static byte[] getStateColumn(int replicaId) { +329 return replicaId == 0 +330 ? HConstants.STATE_QUALIFIER +331 : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER +332 + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId)); +333 } +334}