Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7F6CD106BB for ; Thu, 6 Feb 2014 02:05:23 +0000 (UTC) Received: (qmail 42443 invoked by uid 500); 6 Feb 2014 02:05:22 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 42181 invoked by uid 500); 6 Feb 2014 02:05:22 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 42001 invoked by uid 99); 6 Feb 2014 02:05:20 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Feb 2014 02:05:20 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Feb 2014 02:05:15 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C0CEB2388A36; Thu, 6 Feb 2014 02:04:55 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1565041 [2/3] - in /hbase/branches/hbase-10070: hbase-client/src/main/java/org/apache/hadoop/hbase/ hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ hbase-client/src/tes... Date: Thu, 06 Feb 2014 02:04:54 -0000 To: commits@hbase.apache.org From: enis@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140206020455.C0CEB2388A36@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Feb 6 02:04:53 2014 @@ -35,9 +35,6 @@ import java.util.NavigableMap; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -57,6 +54,7 @@ import org.apache.hadoop.hbase.HRegionIn import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; @@ -64,6 +62,7 @@ import org.apache.hadoop.hbase.TableName import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; @@ -77,7 +76,87 @@ import org.apache.hadoop.hbase.protobuf. import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; @@ -402,6 +481,7 @@ public class HConnectionManager { * @param conf configuration whose identity is used to find {@link HConnection} instance. * @deprecated */ + @Deprecated public static void deleteConnection(Configuration conf) { deleteConnection(new HConnectionKey(conf), false); } @@ -413,6 +493,7 @@ public class HConnectionManager { * @param connection * @deprecated */ + @Deprecated public static void deleteStaleConnection(HConnection connection) { deleteConnection(connection, true); } @@ -423,6 +504,7 @@ public class HConnectionManager { * staleConnection to true. * @deprecated */ + @Deprecated public static void deleteAllConnections(boolean staleConnection) { synchronized (CONNECTION_INSTANCES) { Set connectionKeys = new HashSet(); @@ -479,12 +561,14 @@ public class HConnectionManager { * @return Number of cached regions for the table. * @throws ZooKeeperConnectionException */ + @VisibleForTesting static int getCachedRegionCount(Configuration conf, final TableName tableName) throws IOException { return execute(new HConnectable(conf) { @Override public Integer connect(HConnection connection) { - return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName); + return ((HConnectionImplementation)connection).metaCache + .getNumberOfCachedRegionLocations(tableName); } }); } @@ -495,6 +579,7 @@ public class HConnectionManager { * @return true if the region where the table and row reside is cached. * @throws ZooKeeperConnectionException */ + @VisibleForTesting static boolean isRegionCached(Configuration conf, final TableName tableName, final byte[] row) @@ -502,7 +587,7 @@ public class HConnectionManager { return execute(new HConnectable(conf) { @Override public Boolean connect(HConnection connection) { - return ((HConnectionImplementation) connection).isRegionCached(tableName, row); + return ((HConnectionImplementation) connection).metaCache.isRegionCached(tableName, row); } }); } @@ -584,24 +669,7 @@ public class HConnectionManager { // Client rpc instance. private RpcClient rpcClient; - /** - * Map of table to table {@link HRegionLocation}s. - */ - private final ConcurrentMap> - cachedRegionLocations = - new ConcurrentHashMap>(); - - // The presence of a server in the map implies it's likely that there is an - // entry in cachedRegionLocations that map to this server; but the absence - // of a server in this map guarentees that there is no entry in cache that - // maps to the absent server. - // The access to this attribute must be protected by a lock on cachedRegionLocations - private final Set cachedServers = new ConcurrentSkipListSet(); - - // region cache prefetch is enabled by default. this set contains all - // tables whose region cache prefetch are disabled. - private final Set regionCachePrefetchDisabledTables = - new CopyOnWriteArraySet(); + private MetaCache metaCache = new MetaCache(); private int refCount; @@ -813,6 +881,7 @@ public class HConnectionManager { * An identifier that will remain the same for a given connection. * @return */ + @Override public String toString(){ return "hconnection-0x" + Integer.toHexString(hashCode()); } @@ -984,8 +1053,9 @@ public class HConnectionManager { @Override public HRegionLocation locateRegion(final byte[] regionName) throws IOException { - return locateRegion(HRegionInfo.getTable(regionName), + RegionLocations locations = locateRegion(HRegionInfo.getTable(regionName), HRegionInfo.getStartKey(regionName), false, true); + return locations == null ? null : locations.getRegionLocation(); } @Override @@ -1016,7 +1086,15 @@ public class HConnectionManager { tableName, offlined); final List locations = new ArrayList(); for (HRegionInfo regionInfo : regions.keySet()) { - locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true)); + RegionLocations list = locateRegion(tableName, regionInfo.getStartKey(), useCache, true); + if (list != null) { + for (HRegionLocation loc : list.getRegionLocations()) { + if (loc != null) { + locations.add(loc); + } + } + } + } return locations; } @@ -1031,7 +1109,8 @@ public class HConnectionManager { public HRegionLocation locateRegion(final TableName tableName, final byte [] row) throws IOException{ - return locateRegion(tableName, row, true, true); + RegionLocations locations = locateRegion(tableName, row, true, true); + return locations == null ? null : locations.getRegionLocation(); } @Override @@ -1051,7 +1130,8 @@ public class HConnectionManager { throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled."); } - return locateRegion(tableName, row, false, true); + RegionLocations locations = locateRegion(tableName, row, false, true); + return locations == null ? null : locations.getRegionLocation(); } @Override @@ -1061,7 +1141,7 @@ public class HConnectionManager { } - private HRegionLocation locateRegion(final TableName tableName, + private RegionLocations locateRegion(final TableName tableName, final byte [] row, boolean useCache, boolean retry) throws IOException { if (this.closed) throw new IOException(toString() + " closed"); @@ -1089,12 +1169,14 @@ public class HConnectionManager { // Implement a new visitor for MetaScanner, and use it to walk through // the hbase:meta MetaScannerVisitor visitor = new MetaScannerVisitorBase() { + @Override public boolean processRow(Result result) throws IOException { try { - HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result); - if (regionInfo == null) { + RegionLocations locations = MetaReader.getRegionLocations(result); + if (locations == null) { return true; } + HRegionInfo regionInfo = locations.getRegionLocation().getRegionInfo(); // possible we got a region of a different table... if (!regionInfo.getTable().equals(tableName)) { @@ -1105,15 +1187,13 @@ public class HConnectionManager { return true; } - ServerName serverName = HRegionInfo.getServerName(result); + ServerName serverName = locations.getRegionLocation().getServerName(); if (serverName == null) { return true; // don't cache it } - // instantiate the location - long seqNum = HRegionInfo.getSeqNumDuringOpen(result); - HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum); + // cache this meta entry - cacheLocation(tableName, null, loc); + cacheLocation(tableName, locations); return true; } catch (RuntimeException e) { throw new IOException(e); @@ -1137,11 +1217,11 @@ public class HConnectionManager { * Search the hbase:meta table for the HRegionLocation * info that contains the table and row we're seeking. */ - private HRegionLocation locateRegionInMeta(final TableName parentTable, + private RegionLocations locateRegionInMeta(final TableName parentTable, final TableName tableName, final byte [] row, boolean useCache, Object regionLockObject, boolean retry) throws IOException { - HRegionLocation location; + RegionLocations location; // If we are supposed to be using the cache, look in the cache to see if // we already have the region. if (useCache) { @@ -1165,7 +1245,8 @@ public class HConnectionManager { HRegionLocation metaLocation = null; try { // locate the meta region - metaLocation = locateRegion(parentTable, metaKey, true, false); + RegionLocations metaLocations = locateRegion(parentTable, metaKey, true, false); + metaLocation = metaLocations == null ? null : metaLocations.getRegionLocation(); // If null still, go around again. if (metaLocation == null) continue; ClientService.BlockingInterface service = getClient(metaLocation.getServerName()); @@ -1196,7 +1277,7 @@ public class HConnectionManager { } else { // If we are not supposed to be using the cache, delete any existing cached location // so it won't interfere. - forceDeleteCachedLocation(tableName, row); + metaCache.clearCache(tableName, row); } // Query the meta region for the location of the meta region @@ -1209,7 +1290,8 @@ public class HConnectionManager { } // convert the row result into the HRegionLocation we need! - HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow); + location = MetaReader.getRegionLocations(regionInfoRow); + HRegionInfo regionInfo = location.getRegionLocation().getRegionInfo(); if (regionInfo == null) { throw new IOException("HRegionInfo was null or empty in " + parentTable + ", row=" + regionInfoRow); @@ -1233,7 +1315,7 @@ public class HConnectionManager { regionInfo.getRegionNameAsString()); } - ServerName serverName = HRegionInfo.getServerName(regionInfoRow); + ServerName serverName = location.getRegionLocation().getServerName(); if (serverName == null) { throw new NoServerForRegionException("No server address listed " + "in " + parentTable + " for region " + @@ -1247,10 +1329,7 @@ public class HConnectionManager { ", but it is dead."); } - // Instantiate the location - location = new HRegionLocation(regionInfo, serverName, - HRegionInfo.getSeqNumDuringOpen(regionInfoRow)); - cacheLocation(tableName, null, location); + cacheLocation(tableName, location); return location; } catch (TableNotFoundException e) { // if we got this error, probably means the table just plain doesn't @@ -1290,7 +1369,16 @@ public class HConnectionManager { } } - /* + /** + * Put a newly discovered HRegionLocation into the cache. + * @param tableName The table name. + * @param location the new location + */ + private void cacheLocation(final TableName tableName, final RegionLocations location) { + metaCache.cacheLocation(tableName, location); + } + + /** * Search the cache for a location that fits our table and row key. * Return null if no suitable region is located. * @@ -1298,31 +1386,9 @@ public class HConnectionManager { * @param row * @return Null or region location found in cache. */ - HRegionLocation getCachedLocation(final TableName tableName, + RegionLocations getCachedLocation(final TableName tableName, final byte [] row) { - ConcurrentSkipListMap tableLocations = - getTableLocations(tableName); - - Entry e = tableLocations.floorEntry(row); - if (e == null) { - return null; - } - HRegionLocation possibleRegion = e.getValue(); - - // make sure that the end key is greater than the row we're looking - // for, otherwise the row actually belongs in the next region, not - // this one. the exception case is when the endkey is - // HConstants.EMPTY_END_ROW, signifying that the region we're - // checking is actually the last region in the table. - byte[] endKey = possibleRegion.getRegionInfo().getEndKey(); - if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) || - tableName.getRowComparator().compareRows( - endKey, 0, endKey.length, row, 0, row.length) > 0) { - return possibleRegion; - } - - // Passed all the way through, so we got nothing - complete cache miss - return null; + return metaCache.getCachedLocation(tableName, row); } /** @@ -1330,20 +1396,8 @@ public class HConnectionManager { * @param tableName tableName * @param row */ - void forceDeleteCachedLocation(final TableName tableName, final byte [] row) { - HRegionLocation rl = null; - Map tableLocations = getTableLocations(tableName); - // start to examine the cache. we can only do cache actions - // if there's something in the cache for this table. - rl = getCachedLocation(tableName, row); - if (rl != null) { - tableLocations.remove(rl.getRegionInfo().getStartKey()); - } - if ((rl != null) && LOG.isDebugEnabled()) { - LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort() - + " as a location of " + rl.getRegionInfo().getRegionNameAsString() + - " for tableName=" + tableName + " from cache"); - } + public void clearRegionCache(final TableName tableName, byte[] row) { + metaCache.clearCache(tableName, row); } /* @@ -1351,66 +1405,17 @@ public class HConnectionManager { */ @Override public void clearCaches(final ServerName serverName) { - if (!this.cachedServers.contains(serverName)) { - return; - } - - boolean deletedSomething = false; - synchronized (this.cachedServers) { - // We block here, because if there is an error on a server, it's likely that multiple - // threads will get the error simultaneously. If there are hundreds of thousand of - // region location to check, it's better to do this only once. A better pattern would - // be to check if the server is dead when we get the region location. - if (!this.cachedServers.contains(serverName)) { - return; - } - for (Map tableLocations : cachedRegionLocations.values()) { - for (Entry e : tableLocations.entrySet()) { - HRegionLocation value = e.getValue(); - if (value != null - && serverName.equals(value.getServerName())) { - tableLocations.remove(e.getKey()); - deletedSomething = true; - } - } - } - this.cachedServers.remove(serverName); - } - if (deletedSomething && LOG.isDebugEnabled()) { - LOG.debug("Removed all cached region locations that map to " + serverName); - } - } - - /* - * @param tableName - * @return Map of cached locations for passed tableName - */ - private ConcurrentSkipListMap getTableLocations( - final TableName tableName) { - // find the map of cached locations for this table - ConcurrentSkipListMap result; - result = this.cachedRegionLocations.get(tableName); - // if tableLocations for this table isn't built yet, make one - if (result == null) { - result = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); - ConcurrentSkipListMap old = - this.cachedRegionLocations.putIfAbsent(tableName, result); - if (old != null) { - return old; - } - } - return result; + metaCache.clearCache(serverName); } @Override public void clearRegionCache() { - this.cachedRegionLocations.clear(); - this.cachedServers.clear(); + metaCache.clearCache(); } @Override public void clearRegionCache(final TableName tableName) { - this.cachedRegionLocations.remove(tableName); + metaCache.clearCache(tableName); } @Override @@ -1426,37 +1431,7 @@ public class HConnectionManager { */ private void cacheLocation(final TableName tableName, final ServerName source, final HRegionLocation location) { - boolean isFromMeta = (source == null); - byte [] startKey = location.getRegionInfo().getStartKey(); - ConcurrentMap tableLocations = getTableLocations(tableName); - HRegionLocation oldLocation = tableLocations.putIfAbsent(startKey, location); - boolean isNewCacheEntry = (oldLocation == null); - if (isNewCacheEntry) { - cachedServers.add(location.getServerName()); - return; - } - boolean updateCache; - // If the server in cache sends us a redirect, assume it's always valid. - if (oldLocation.getServerName().equals(source)) { - updateCache = true; - } else { - long newLocationSeqNum = location.getSeqNum(); - // Meta record is stale - some (probably the same) server has closed the region - // with later seqNum and told us about the new location. - boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum); - // Same as above for redirect. However, in this case, if the number is equal to previous - // record, the most common case is that first the region was closed with seqNum, and then - // opened with the same seqNum; hence we will ignore the redirect. - // There are so many corner cases with various combinations of opens and closes that - // an additional counter on top of seqNum would be necessary to handle them all. - boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum); - boolean isStaleUpdate = (isStaleMetaRecord || isStaleRedirect); - updateCache = (!isStaleUpdate); - } - if (updateCache) { - tableLocations.replace(startKey, oldLocation, location); - cachedServers.add(location.getServerName()); - } + metaCache.cacheLocation(tableName, source, location); } // Map keyed by service name + regionserver to service stub implementation @@ -2120,7 +2095,7 @@ public class HConnectionManager { } }; } - + private static void release(MasterServiceState mss) { if (mss != null && mss.connection != null) { @@ -2179,37 +2154,17 @@ public class HConnectionManager { cacheLocation(hri.getTable(), source, newHrl); } - /** - * Deletes the cached location of the region if necessary, based on some error from source. - * @param hri The region in question. - * @param source The source of the error that prompts us to invalidate cache. - */ - void deleteCachedLocation(HRegionInfo hri, ServerName source) { - getTableLocations(hri.getTable()).remove(hri.getStartKey()); - } - @Override public void deleteCachedRegionLocation(final HRegionLocation location) { - if (location == null || location.getRegionInfo() == null) { - return; - } - - HRegionLocation removedLocation; - TableName tableName = location.getRegionInfo().getTable(); - Map tableLocations = getTableLocations(tableName); - removedLocation = tableLocations.remove(location.getRegionInfo().getStartKey()); - if (LOG.isDebugEnabled() && removedLocation != null) { - LOG.debug("Removed " + - location.getRegionInfo().getRegionNameAsString() + - " for tableName=" + tableName + - " from cache"); - } + metaCache.clearCache(location); } @Override public void updateCachedLocations(final TableName tableName, byte[] rowkey, final Object exception, final HRegionLocation source) { - updateCachedLocations(tableName, rowkey, exception, source.getServerName()); + assert source != null; + updateCachedLocations(tableName, source.getRegionInfo().getRegionName() + , rowkey, exception, source.getServerName()); } /** @@ -2221,7 +2176,7 @@ public class HConnectionManager { * @param source server that is the source of the location update. */ @Override - public void updateCachedLocations(final TableName tableName, byte[] rowkey, + public void updateCachedLocations(final TableName tableName, byte[] regionName, byte[] rowkey, final Object exception, final ServerName source) { if (rowkey == null || tableName == null) { LOG.warn("Coding error, see method javadoc. row=" + (rowkey == null ? "null" : rowkey) + @@ -2234,8 +2189,18 @@ public class HConnectionManager { return; } + if (regionName == null) { + // we do not know which region, so just remove the cache entry for the row and server + metaCache.clearCache(tableName, rowkey, source); + return; + } + // Is it something we have already updated? - final HRegionLocation oldLocation = getCachedLocation(tableName, rowkey); + final RegionLocations oldLocations = getCachedLocation(tableName, rowkey); + HRegionLocation oldLocation = null; + if (oldLocations != null) { + oldLocation = oldLocations.getRegionLocationByRegionName(regionName); + } if (oldLocation == null || !source.equals(oldLocation.getServerName())) { // There is no such location in the cache (it's been removed already) or // the cache has already been refreshed with a different location. => nothing to do @@ -2266,8 +2231,8 @@ public class HConnectionManager { } // If we're here, it means that can cannot be sure about the location, so we remove it from - // the cache. - deleteCachedLocation(regionInfo, source); + // the cache. Do not send the source because source can be a new server in the same host:port + metaCache.clearCache(regionInfo); } @Override @@ -2354,35 +2319,15 @@ public class HConnectionManager { * Return the number of cached region for a table. It will only be called * from a unit test. */ + @VisibleForTesting int getNumberOfCachedRegionLocations(final TableName tableName) { - Map tableLocs = this.cachedRegionLocations.get(tableName); - if (tableLocs == null) { - return 0; - } - return tableLocs.values().size(); - } - - /** - * Check the region cache to see whether a region is cached yet or not. - * Called by unit tests. - * @param tableName tableName - * @param row row - * @return Region cached or not. - */ - boolean isRegionCached(TableName tableName, final byte[] row) { - HRegionLocation location = getCachedLocation(tableName, row); - return location != null; + return metaCache.getNumberOfCachedRegionLocations(tableName); } @Override public void setRegionCachePrefetch(final TableName tableName, final boolean enable) { - if (!enable) { - regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName())); - } - else { - regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName())); - } + metaCache.setRegionCachePrefetch(tableName, enable); } @Override @@ -2393,7 +2338,7 @@ public class HConnectionManager { @Override public boolean getRegionCachePrefetch(TableName tableName) { - return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName())); + return metaCache.getRegionCachePrefetch(tableName); } @Override @@ -2701,7 +2646,7 @@ public class HConnectionManager { * Look for an exception we know in the remote exception: * - hadoop.ipc wrapped exceptions * - nested exceptions - * + * * Looks for: RegionMovedException / RegionOpeningException / RegionTooBusyException * @return null if we didn't find the exception, the exception otherwise. */ Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java Thu Feb 6 02:04:53 2014 @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.HRegionLo import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture; @@ -69,6 +70,7 @@ import org.apache.hadoop.hbase.util.Byte import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; @@ -585,12 +587,15 @@ public class HTable implements HTableInt * @return Pair of arrays of region starting and ending row keys * @throws IOException if a remote or network exception occurs */ + // TODO: these are not in HTableInterface. Should we add them there or move these to HBaseAdmin? public Pair getStartEndKeys() throws IOException { - NavigableMap regions = getRegionLocations(); + + List regions = listRegionLocations(); final List startKeyList = new ArrayList(regions.size()); final List endKeyList = new ArrayList(regions.size()); - for (HRegionInfo region : regions.keySet()) { + for (RegionLocations locations : regions) { + HRegionInfo region = locations.getRegionLocation().getRegionInfo(); startKeyList.add(region.getStartKey()); endKeyList.add(region.getEndKey()); } @@ -600,13 +605,20 @@ public class HTable implements HTableInt endKeyList.toArray(new byte[endKeyList.size()][])); } + @VisibleForTesting + List listRegionLocations() throws IOException { + return MetaScanner.listTableRegionLocations(getConfiguration(), this.connection, getName()); + } + /** * Gets all the regions and their address for this table. *

* This is mainly useful for the MapReduce integration. * @return A map of HRegionInfo with it's server address * @throws IOException if a remote or network exception occurs + * @deprecated This is no longer a public API */ + @Deprecated public NavigableMap getRegionLocations() throws IOException { // TODO: Odd that this returns a Map of HRI to SN whereas getRegionLocation, singular, returns an HRegionLocation. return MetaScanner.allTableRegions(getConfiguration(), this.connection, getName(), false); @@ -620,7 +632,9 @@ public class HTable implements HTableInt * @return A list of HRegionLocations corresponding to the regions that * contain the specified range * @throws IOException if a remote or network exception occurs + * @deprecated This is no longer a public API */ + @Deprecated public List getRegionsInRange(final byte [] startKey, final byte [] endKey) throws IOException { return getRegionsInRange(startKey, endKey, false); @@ -635,7 +649,9 @@ public class HTable implements HTableInt * @return A list of HRegionLocations corresponding to the regions that * contain the specified range * @throws IOException if a remote or network exception occurs + * @deprecated This is no longer a public API */ + @Deprecated public List getRegionsInRange(final byte [] startKey, final byte [] endKey, final boolean reload) throws IOException { return getKeysAndRegionsInRange(startKey, endKey, false, reload).getSecond(); @@ -651,7 +667,9 @@ public class HTable implements HTableInt * @return A pair of list of start keys and list of HRegionLocations that * contain the specified range * @throws IOException if a remote or network exception occurs + * @deprecated This is no longer a public API */ + @Deprecated private Pair, List> getKeysAndRegionsInRange( final byte[] startKey, final byte[] endKey, final boolean includeEndKey) throws IOException { @@ -669,7 +687,9 @@ public class HTable implements HTableInt * @return A pair of list of start keys and list of HRegionLocations that * contain the specified range * @throws IOException if a remote or network exception occurs + * @deprecated This is no longer a public API */ + @Deprecated private Pair, List> getKeysAndRegionsInRange( final byte[] startKey, final byte[] endKey, final boolean includeEndKey, final boolean reload) throws IOException { @@ -702,7 +722,8 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(this.connection, tableName, row) { - public Result call() throws IOException { + @Override + public Result call() throws IOException { return ProtobufUtil.getRowOrBefore(getStub(), getLocation().getRegionInfo().getRegionName(), row, family); } @@ -757,6 +778,7 @@ public class HTable implements HTableInt public Result get(final Get get) throws IOException { RegionServerCallable callable = new RegionServerCallable(this.connection, getName(), get.getRow()) { + @Override public Result call() throws IOException { return ProtobufUtil.get(getStub(), getLocation().getRegionInfo().getRegionName(), get); } @@ -807,6 +829,7 @@ public class HTable implements HTableInt * @deprecated If any exception is thrown by one of the actions, there is no way to * retrieve the partially executed results. Use {@link #batch(List, Object[])} instead. */ + @Deprecated @Override public Object[] batch(final List actions) throws InterruptedException, IOException { @@ -832,6 +855,7 @@ public class HTable implements HTableInt * {@link #batchCallback(List, Object[], org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} * instead. */ + @Deprecated @Override public Object[] batchCallback( final List actions, final Batch.Callback callback) throws IOException, @@ -849,6 +873,7 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, tableName, delete.getRow()) { + @Override public Boolean call() throws IOException { try { MutateRequest request = RequestConverter.buildMutateRequest( @@ -985,6 +1010,7 @@ public class HTable implements HTableInt public void mutateRow(final RowMutations rm) throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, getName(), rm.getRow()) { + @Override public Void call() throws IOException { try { RegionAction.Builder regionMutationBuilder = RequestConverter.buildRegionAction( @@ -1018,6 +1044,7 @@ public class HTable implements HTableInt final long nonceGroup = ng.getNonceGroup(), nonce = ng.newNonce(); RegionServerCallable callable = new RegionServerCallable(this.connection, getName(), append.getRow()) { + @Override public Result call() throws IOException { try { MutateRequest request = RequestConverter.buildMutateRequest( @@ -1048,6 +1075,7 @@ public class HTable implements HTableInt final long nonceGroup = ng.getNonceGroup(), nonce = ng.newNonce(); RegionServerCallable callable = new RegionServerCallable(this.connection, getName(), increment.getRow()) { + @Override public Result call() throws IOException { try { MutateRequest request = RequestConverter.buildMutateRequest( @@ -1110,6 +1138,7 @@ public class HTable implements HTableInt final long nonceGroup = ng.getNonceGroup(), nonce = ng.newNonce(); RegionServerCallable callable = new RegionServerCallable(connection, getName(), row) { + @Override public Long call() throws IOException { try { MutateRequest request = RequestConverter.buildIncrementRequest( @@ -1139,6 +1168,7 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, getName(), row) { + @Override public Boolean call() throws IOException { try { MutateRequest request = RequestConverter.buildMutateRequest( @@ -1164,6 +1194,7 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, getName(), row) { + @Override public Boolean call() throws IOException { try { CompareType compareType = CompareType.valueOf(compareOp.name()); @@ -1190,6 +1221,7 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, getName(), row) { + @Override public Boolean call() throws IOException { try { MutateRequest request = RequestConverter.buildMutateRequest( @@ -1215,6 +1247,7 @@ public class HTable implements HTableInt throws IOException { RegionServerCallable callable = new RegionServerCallable(connection, getName(), row) { + @Override public Boolean call() throws IOException { try { CompareType compareType = CompareType.valueOf(compareOp.name()); @@ -1399,6 +1432,7 @@ public class HTable implements HTableInt * @param writeBufferSize The new write buffer size, in bytes. * @throws IOException if a remote or network exception occurs. */ + @Override public void setWriteBufferSize(long writeBufferSize) throws IOException { this.writeBufferSize = writeBufferSize; if(currentWriteBufferSize > writeBufferSize) { @@ -1524,6 +1558,7 @@ public class HTable implements HTableInt /** * {@inheritDoc} */ + @Override public CoprocessorRpcChannel coprocessorService(byte[] row) { return new RegionCoprocessorRpcChannel(connection, tableName, row); } @@ -1538,6 +1573,7 @@ public class HTable implements HTableInt final Map results = Collections.synchronizedMap( new TreeMap(Bytes.BYTES_COMPARATOR)); coprocessorService(service, startKey, endKey, callable, new Batch.Callback() { + @Override public void update(byte[] region, byte[] row, R value) { if (region != null) { results.put(region, value); @@ -1565,6 +1601,7 @@ public class HTable implements HTableInt new RegionCoprocessorRpcChannel(connection, tableName, r); Future future = pool.submit( new Callable() { + @Override public R call() throws Exception { T instance = ProtobufUtil.newServiceStub(service, channel); R result = callable.call(instance); Added: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java?rev=1565041&view=auto ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java (added) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java Thu Feb 6 02:04:53 2014 @@ -0,0 +1,363 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArraySet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A cache implementation for region locations from meta. + */ +@InterfaceAudience.Private +public class MetaCache { + + private static final Log LOG = LogFactory.getLog(MetaCache.class); + + /** + * Map of table to table {@link HRegionLocation}s. + */ + private final ConcurrentMap> + cachedRegionLocations = + new ConcurrentHashMap>(); + + // The presence of a server in the map implies it's likely that there is an + // entry in cachedRegionLocations that map to this server; but the absence + // of a server in this map guarentees that there is no entry in cache that + // maps to the absent server. + // The access to this attribute must be protected by a lock on cachedRegionLocations + private final Set cachedServers = new ConcurrentSkipListSet(); + + // region cache prefetch is enabled by default. this set contains all + // tables whose region cache prefetch are disabled. + private final Set regionCachePrefetchDisabledTables = + new CopyOnWriteArraySet(); + + /** + * Search the cache for a location that fits our table and row key. + * Return null if no suitable region is located. + * + * @param tableName + * @param row + * @return Null or region location found in cache. + */ + public RegionLocations getCachedLocation(final TableName tableName, final byte [] row) { + ConcurrentSkipListMap tableLocations = + getTableLocations(tableName); + + Entry e = tableLocations.floorEntry(row); + if (e == null) { + return null; + } + RegionLocations possibleRegion = e.getValue(); + + // make sure that the end key is greater than the row we're looking + // for, otherwise the row actually belongs in the next region, not + // this one. the exception case is when the endkey is + // HConstants.EMPTY_END_ROW, signifying that the region we're + // checking is actually the last region in the table. + byte[] endKey = possibleRegion.getRegionLocation().getRegionInfo().getEndKey(); + if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) || + tableName.getRowComparator().compareRows( + endKey, 0, endKey.length, row, 0, row.length) > 0) { + return possibleRegion; + } + + // Passed all the way through, so we got nothing - complete cache miss + return null; + } + + /** + * Put a newly discovered HRegionLocation into the cache. + * @param tableName The table name. + * @param source the source of the new location + * @param location the new location + */ + public void cacheLocation(final TableName tableName, final ServerName source, + final HRegionLocation location) { + assert source != null; + byte [] startKey = location.getRegionInfo().getStartKey(); + ConcurrentMap tableLocations = getTableLocations(tableName); + RegionLocations locations = new RegionLocations(new HRegionLocation[] {location}) ; + RegionLocations oldLocations = tableLocations.putIfAbsent(startKey, locations); + boolean isNewCacheEntry = (oldLocations == null); + if (isNewCacheEntry) { + addToCachedServers(locations); + return; + } + + // If the server in cache sends us a redirect, assume it's always valid. + HRegionLocation oldLocation = oldLocations.getRegionLocation( + location.getRegionInfo().getReplicaId()); + boolean force = oldLocation != null && oldLocation.getServerName() != null + && oldLocation.getServerName().equals(source); + + // For redirect if the number is equal to previous + // record, the most common case is that first the region was closed with seqNum, and then + // opened with the same seqNum; hence we will ignore the redirect. + // There are so many corner cases with various combinations of opens and closes that + // an additional counter on top of seqNum would be necessary to handle them all. + RegionLocations updatedLocations = oldLocations.updateLocation(location, false, force); + if (oldLocations != updatedLocations) { + tableLocations.replace(startKey, oldLocations, updatedLocations); + addToCachedServers(updatedLocations); + } + } + + /** + * Put a newly discovered HRegionLocation into the cache. + * @param tableName The table name. + * @param location the new location + */ + public void cacheLocation(final TableName tableName, final RegionLocations location) { + byte [] startKey = location.getRegionLocation().getRegionInfo().getStartKey(); + ConcurrentMap tableLocations = getTableLocations(tableName); + RegionLocations oldLocation = tableLocations.putIfAbsent(startKey, location); + boolean isNewCacheEntry = (oldLocation == null); + if (isNewCacheEntry) { + addToCachedServers(location); + return; + } + + // merge old and new locations and add it to the cache + // Meta record might be stale - some (probably the same) server has closed the region + // with later seqNum and told us about the new location. + RegionLocations mergedLocation = oldLocation.mergeLocations(location); + tableLocations.replace(startKey, oldLocation, mergedLocation); + addToCachedServers(location); + } + + private void addToCachedServers(RegionLocations locations) { + for (HRegionLocation loc : locations.getRegionLocations()) { + if (loc != null) { + cachedServers.add(loc.getServerName()); + } + } + } + + /** + * @param tableName + * @return Map of cached locations for passed tableName + */ + private ConcurrentSkipListMap + getTableLocations(final TableName tableName) { + // find the map of cached locations for this table + ConcurrentSkipListMap result; + result = this.cachedRegionLocations.get(tableName); + // if tableLocations for this table isn't built yet, make one + if (result == null) { + result = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + ConcurrentSkipListMap old = + this.cachedRegionLocations.putIfAbsent(tableName, result); + if (old != null) { + return old; + } + } + return result; + } + + /** + * Check the region cache to see whether a region is cached yet or not. + * @param tableName tableName + * @param row row + * @return Region cached or not. + */ + public boolean isRegionCached(TableName tableName, final byte[] row) { + RegionLocations location = getCachedLocation(tableName, row); + return location != null; + } + + /** + * Return the number of cached region for a table. It will only be called + * from a unit test. + */ + public int getNumberOfCachedRegionLocations(final TableName tableName) { + Map tableLocs = this.cachedRegionLocations.get(tableName); + if (tableLocs == null) { + return 0; + } + int numRegions = 0; + for (RegionLocations tableLoc : tableLocs.values()) { + numRegions += tableLoc.numNonNullElements(); + } + return numRegions; + } + + /** + * Delete all cached entries. + */ + public void clearCache() { + this.cachedRegionLocations.clear(); + this.cachedServers.clear(); + } + + /** + * Delete all cached entries of a server. + */ + public void clearCache(final ServerName serverName) { + if (!this.cachedServers.contains(serverName)) { + return; + } + + boolean deletedSomething = false; + synchronized (this.cachedServers) { + // We block here, because if there is an error on a server, it's likely that multiple + // threads will get the error simultaneously. If there are hundreds of thousand of + // region location to check, it's better to do this only once. A better pattern would + // be to check if the server is dead when we get the region location. + if (!this.cachedServers.contains(serverName)) { + return; + } + for (ConcurrentMap tableLocations : cachedRegionLocations.values()){ + for (Entry e : tableLocations.entrySet()) { + RegionLocations regionLocations = e.getValue(); + if (regionLocations != null) { + RegionLocations updatedLocations = regionLocations.removeByServer(serverName); + deletedSomething |= regionLocations == updatedLocations; + if (updatedLocations != regionLocations) { + if (updatedLocations.isEmpty()) { + tableLocations.remove(e.getKey(), regionLocations); + } else { + tableLocations.replace(e.getKey(), regionLocations, updatedLocations); + } + } + } + } + } + this.cachedServers.remove(serverName); + } + if (deletedSomething && LOG.isDebugEnabled()) { + LOG.debug("Removed all cached region locations that map to " + serverName); + } + } + + /** + * Delete all cached entries of a table. + */ + public void clearCache(final TableName tableName) { + this.cachedRegionLocations.remove(tableName); + } + + /** + * Delete a cached location, no matter what it is. Called when we were told to not use cache. + * @param tableName tableName + * @param row + */ + public void clearCache(final TableName tableName, final byte [] row) { + ConcurrentMap tableLocations = getTableLocations(tableName); + + RegionLocations regionLocations = getCachedLocation(tableName, row); + if (regionLocations != null) { + byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey(); + boolean removed = tableLocations.remove(startKey, regionLocations); + if (removed && LOG.isDebugEnabled()) { + LOG.debug("Removed " + regionLocations + " from cache"); + } + } + } + + /** + * Delete a cached location for a table, row and server + */ + public void clearCache(final TableName tableName, final byte [] row, ServerName serverName) { + ConcurrentMap tableLocations = getTableLocations(tableName); + + RegionLocations regionLocations = getCachedLocation(tableName, row); + if (regionLocations != null) { + RegionLocations updatedLocations = regionLocations.removeByServer(serverName); + if (updatedLocations != regionLocations) { + byte[] startKey = regionLocations.getRegionLocation().getRegionInfo().getStartKey(); + if (updatedLocations.isEmpty()) { + tableLocations.remove(startKey, regionLocations); + } else { + tableLocations.replace(startKey, regionLocations, updatedLocations); + } + } + } + } + + /** + * Deletes the cached location of the region if necessary, based on some error from source. + * @param hri The region in question. + */ + public void clearCache(HRegionInfo hri) { + ConcurrentMap tableLocations = getTableLocations(hri.getTable()); + RegionLocations regionLocations = tableLocations.get(hri.getStartKey()); + if (regionLocations != null) { + HRegionLocation oldLocation = regionLocations.getRegionLocation(hri.getReplicaId()); + RegionLocations updatedLocations = regionLocations.remove(oldLocation); + if (updatedLocations != regionLocations) { + if (updatedLocations.isEmpty()) { + tableLocations.remove(hri.getStartKey(), regionLocations); + } else { + tableLocations.replace(hri.getStartKey(), regionLocations, updatedLocations); + } + } + } + } + + public void clearCache(final HRegionLocation location) { + if (location == null) { + return; + } + + TableName tableName = location.getRegionInfo().getTable(); + ConcurrentMap tableLocations = getTableLocations(tableName); + RegionLocations rll = tableLocations.get(location.getRegionInfo().getStartKey()); + RegionLocations updatedLocations = rll.remove(location); + if (updatedLocations.isEmpty()) { + tableLocations.remove(location.getRegionInfo().getStartKey(), rll); + } + if (LOG.isDebugEnabled() && (rll == updatedLocations)) { + LOG.debug("Removed " + + location.getRegionInfo().getRegionNameAsString() + + " for tableName=" + tableName + + " from cache"); + } + } + + public void setRegionCachePrefetch(final TableName tableName, final boolean enable) { + if (!enable) { + regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName())); + } else { + regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName())); + } + } + + public boolean getRegionCachePrefetch(TableName tableName) { + return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName())); + } + +} Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java Thu Feb 6 02:04:53 2014 @@ -30,11 +30,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.util.Bytes; /** @@ -49,6 +52,7 @@ import org.apache.hadoop.hbase.util.Byte * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details.

*/ @InterfaceAudience.Private +//TODO: merge this to MetaReader, get rid of it. public class MetaScanner { private static final Log LOG = LogFactory.getLog(MetaScanner.class); /** @@ -216,14 +220,14 @@ public class MetaScanner { * table Result. * @param data a Result object from the catalog table scan * @return HRegionInfo or null + * @deprecated Use {@link MetaReader#getRegionLocations(Result)} */ + @Deprecated public static HRegionInfo getHRegionInfo(Result data) { return HRegionInfo.getHRegionInfo(data); } /** - * Used in tests. - * * Lists all of the regions currently in META. * @param conf * @param offlined True if we are to include offlined regions, false and we'll @@ -234,22 +238,23 @@ public class MetaScanner { public static List listAllRegions(Configuration conf, final boolean offlined) throws IOException { final List regions = new ArrayList(); - MetaScannerVisitor visitor = new DefaultMetaScannerVisitor() { + MetaScannerVisitor visitor = new MetaScannerVisitorBase() { @Override - public boolean processRowInternal(Result result) throws IOException { + public boolean processRow(Result result) throws IOException { if (result == null || result.isEmpty()) { return true; } - HRegionInfo regionInfo = getHRegionInfo(result); - if (regionInfo == null) { - LOG.warn("Null REGIONINFO_QUALIFIER: " + result); - return true; + RegionLocations locations = MetaReader.getRegionLocations(result); + if (locations == null) return true; + for (HRegionLocation loc : locations.getRegionLocations()) { + if (loc != null) { + HRegionInfo regionInfo = loc.getRegionInfo(); + // If region offline AND we are not to include offlined regions, return. + if (regionInfo.isOffline() && !offlined) continue; + regions.add(regionInfo); + } } - - // If region offline AND we are not to include offlined regions, return. - if (regionInfo.isOffline() && !offlined) return true; - regions.add(regionInfo); return true; } }; @@ -272,10 +277,34 @@ public class MetaScanner { new TreeMap(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override - public boolean processRowInternal(Result rowResult) throws IOException { - HRegionInfo info = getHRegionInfo(rowResult); - ServerName serverName = HRegionInfo.getServerName(rowResult); - regions.put(new UnmodifyableHRegionInfo(info), serverName); + public boolean processRowInternal(Result result) throws IOException { + RegionLocations locations = MetaReader.getRegionLocations(result); + if (locations == null) return true; + for (HRegionLocation loc : locations.getRegionLocations()) { + if (loc != null) { + HRegionInfo regionInfo = loc.getRegionInfo(); + regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName()); + } + } + return true; + } + }; + metaScan(conf, connection, visitor, tableName); + return regions; + } + + /** + * Lists table regions and locations grouped by region range from META. + */ + public static List listTableRegionLocations(Configuration conf, + HConnection connection, final TableName tableName) throws IOException { + final List regions = new ArrayList(); + MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { + @Override + public boolean processRowInternal(Result result) throws IOException { + RegionLocations locations = MetaReader.getRegionLocations(result); + if (locations == null) return true; + regions.add(locations); return true; } }; Added: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java?rev=1565041&view=auto ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java (added) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java Thu Feb 6 02:04:53 2014 @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; + +/** + * Utility methods which contain the logic for regions and replicas. + */ +@InterfaceAudience.Private +public class RegionReplicaUtil { + + /** + * The default replicaId for the region + */ + private static final int DEFAULT_REPLICA_ID = 0; + + /** + * Returns the HRegionInfo for the given replicaId. HRegionInfo's correspond to + * a range of a table, but more than one "instance" of the same range can be + * deployed which are differentiated by the replicaId. + * @param replicaId the replicaId to use + * @return an HRegionInfo object corresponding to the same range (table, start and + * end key), but for the given replicaId. + */ + public static HRegionInfo getRegionInfoForReplica(HRegionInfo regionInfo, int replicaId) { + if (regionInfo.getReplicaId() == replicaId) { + return regionInfo; + } + HRegionInfo replicaInfo = new HRegionInfo(regionInfo.getTable(), regionInfo.getStartKey(), + regionInfo.getEndKey(), regionInfo.isSplit(), regionInfo.getRegionId(), replicaId); + + replicaInfo.setOffline(regionInfo.isOffline()); + return replicaInfo; + } + + /** + * Returns the HRegionInfo for the default replicaId (0). HRegionInfo's correspond to + * a range of a table, but more than one "instance" of the same range can be + * deployed which are differentiated by the replicaId. + * @return an HRegionInfo object corresponding to the same range (table, start and + * end key), but for the default replicaId. + */ + public static HRegionInfo getRegionInfoForDefaultReplica(HRegionInfo regionInfo) { + return getRegionInfoForReplica(regionInfo, DEFAULT_REPLICA_ID); + } + +} Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java Thu Feb 6 02:04:53 2014 @@ -73,6 +73,7 @@ public abstract class RegionServerCallab * @param reload Set this to true if connection should re-find the region * @throws IOException e */ + @Override public void prepare(final boolean reload) throws IOException { this.location = connection.getRegionLocation(tableName, row, reload); if (this.location == null) { @@ -124,7 +125,7 @@ public abstract class RegionServerCallab // hbase:meta again to find the new location if (this.location != null) getConnection().clearCaches(location.getServerName()); } else if (t instanceof RegionMovedException) { - getConnection().updateCachedLocations(tableName, row, t, location.getServerName()); + getConnection().updateCachedLocations(tableName, row, t, location); } else if (t instanceof NotServingRegionException && !retrying) { // Purge cache entries for this specific region from hbase:meta cache // since we don't call connect(true) when number of retries is 1. Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java Thu Feb 6 02:04:53 2014 @@ -19,8 +19,8 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HRegionLocation; /** * Cluster registry. @@ -36,7 +36,7 @@ interface Registry { * @return Meta region location * @throws IOException */ - HRegionLocation getMetaRegionLocation() throws IOException; + RegionLocations getMetaRegionLocation() throws IOException; /** * @return Cluster id. Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java Thu Feb 6 02:04:53 2014 @@ -21,10 +21,11 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; @@ -48,7 +49,7 @@ class ZooKeeperRegistry implements Regis } @Override - public HRegionLocation getMetaRegionLocation() throws IOException { + public RegionLocations getMetaRegionLocation() throws IOException { ZooKeeperKeepAliveConnection zkw = hci.getKeepAliveZooKeeperWatcher(); try { @@ -61,7 +62,8 @@ class ZooKeeperRegistry implements Regis "; serverName=" + ((servername == null) ? "null" : servername)); } if (servername == null) return null; - return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername, 0); + HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername, 0); + return new RegionLocations(new HRegionLocation[] {loc}); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; Added: hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java?rev=1565041&view=auto ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java (added) +++ hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java Thu Feb 6 02:04:53 2014 @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class TestRegionLocations { + + ServerName sn0 = ServerName.valueOf("host0", 10, 10); + ServerName sn1 = ServerName.valueOf("host1", 10, 10); + ServerName sn2 = ServerName.valueOf("host2", 10, 10); + ServerName sn3 = ServerName.valueOf("host3", 10, 10); + + HRegionInfo info0 = hri(0); + HRegionInfo info1 = hri(1); + HRegionInfo info2 = hri(2); + HRegionInfo info9 = hri(9); + + @Test + public void testSizeMethods() { + RegionLocations list = new RegionLocations(); + assertTrue(list.isEmpty()); + assertEquals(0, list.size()); + assertEquals(0, list.numNonNullElements()); + + list = hrll((HRegionLocation)null); + assertTrue(list.isEmpty()); + assertEquals(0, list.size()); + assertEquals(0, list.numNonNullElements()); + + HRegionInfo info0 = hri(0); + list = hrll(hrl(info0, null)); + assertFalse(list.isEmpty()); + assertEquals(1, list.size()); + assertEquals(1, list.numNonNullElements()); + + HRegionInfo info9 = hri(9); + list = hrll(hrl(info9, null)); + assertFalse(list.isEmpty()); + assertEquals(10, list.size()); + assertEquals(1, list.numNonNullElements()); + + list = hrll(hrl(info0, null), hrl(info9, null)); + assertFalse(list.isEmpty()); + assertEquals(10, list.size()); + assertEquals(2, list.numNonNullElements()); + } + + private HRegionInfo hri(int replicaId) { + TableName table = TableName.valueOf("table"); + byte[] startKey = HConstants.EMPTY_START_ROW; + byte[] endKey = HConstants.EMPTY_END_ROW; + long regionId = System.currentTimeMillis(); + HRegionInfo info = new HRegionInfo(table, startKey, endKey, false, regionId, replicaId); + return info; + } + + private HRegionLocation hrl(HRegionInfo hri, ServerName sn) { + return new HRegionLocation(hri, sn); + } + + private HRegionLocation hrl(HRegionInfo hri, ServerName sn, long seqNum) { + return new HRegionLocation(hri, sn, seqNum); + } + + private RegionLocations hrll(HRegionLocation ... locations) { + return new RegionLocations(locations); + } + + @Test + public void testRemoveByServer() { + RegionLocations list; + + // test remove from empty list + list = new RegionLocations(); + assertTrue(list == list.removeByServer(sn0)); + + // test remove from single element list + list = hrll(hrl(info0, sn0)); + assertTrue(list == list.removeByServer(sn1)); + list = list.removeByServer(sn0); + assertTrue(list.isEmpty()); + + // test remove from multi element list + list = hrll(hrl(info0, sn0), hrl(info1, sn1), hrl(info2, sn2), hrl(info9, sn2)); + assertTrue(list == list.removeByServer(sn3)); // no region is mapped to sn3 + list = list.removeByServer(sn0); + assertNull(list.getRegionLocation(0)); + assertEquals(sn1, list.getRegionLocation(1).getServerName()); + assertEquals(sn2, list.getRegionLocation(2).getServerName()); + assertNull(list.getRegionLocation(5)); + assertEquals(sn2, list.getRegionLocation(9).getServerName()); + + // test multi-element remove from multi element list + list = hrll(hrl(info0, sn1), hrl(info1, sn1), hrl(info2, sn0), hrl(info9, sn0)); + list = list.removeByServer(sn0); + assertEquals(sn1, list.getRegionLocation(0).getServerName()); + assertEquals(sn1, list.getRegionLocation(1).getServerName()); + assertNull(list.getRegionLocation(2)); + assertNull(list.getRegionLocation(5)); + assertNull(list.getRegionLocation(9)); + } + + @Test + public void testRemove() { + RegionLocations list; + + // test remove from empty list + list = new RegionLocations(); + assertTrue(list == list.remove(hrl(info0, sn0))); + + // test remove from single element list + list = hrll(hrl(info0, sn0)); + assertTrue(list == list.remove(hrl(info0, sn1))); + list = list.remove(hrl(info0, sn0)); + assertTrue(list.isEmpty()); + + // test remove from multi element list + list = hrll(hrl(info0, sn0), hrl(info1, sn1), hrl(info2, sn2), hrl(info9, sn2)); + assertTrue(list == list.remove(hrl(info1, sn3))); // no region is mapped to sn3 + list = list.remove(hrl(info0, sn0)); + assertNull(list.getRegionLocation(0)); + assertEquals(sn1, list.getRegionLocation(1).getServerName()); + assertEquals(sn2, list.getRegionLocation(2).getServerName()); + assertNull(list.getRegionLocation(5)); + assertEquals(sn2, list.getRegionLocation(9).getServerName()); + + list = list.remove(hrl(info9, sn2)); + assertNull(list.getRegionLocation(0)); + assertEquals(sn1, list.getRegionLocation(1).getServerName()); + assertEquals(sn2, list.getRegionLocation(2).getServerName()); + assertNull(list.getRegionLocation(5)); + assertNull(list.getRegionLocation(9)); + + + // test multi-element remove from multi element list + list = hrll(hrl(info0, sn1), hrl(info1, sn1), hrl(info2, sn0), hrl(info9, sn0)); + list = list.remove(hrl(info9, sn0)); + assertEquals(sn1, list.getRegionLocation(0).getServerName()); + assertEquals(sn1, list.getRegionLocation(1).getServerName()); + assertEquals(sn0, list.getRegionLocation(2).getServerName()); + assertNull(list.getRegionLocation(5)); + assertNull(list.getRegionLocation(9)); + } + + @Test + public void testUpdateLocation() { + RegionLocations list; + + // test add to empty list + list = new RegionLocations(); + list = list.updateLocation(hrl(info0, sn1), false, false); + assertEquals(sn1, list.getRegionLocation(0).getServerName()); + + // test add to non-empty list + list = list.updateLocation(hrl(info9, sn3, 10), false, false); + assertEquals(sn3, list.getRegionLocation(9).getServerName()); + assertEquals(10, list.size()); + list = list.updateLocation(hrl(info2, sn2, 10), false, false); + assertEquals(sn2, list.getRegionLocation(2).getServerName()); + assertEquals(10, list.size()); + + // test update greater SeqNum + list = list.updateLocation(hrl(info2, sn3, 11), false, false); + assertEquals(sn3, list.getRegionLocation(2).getServerName()); + assertEquals(sn3, list.getRegionLocation(9).getServerName()); + + // test update equal SeqNum + list = list.updateLocation(hrl(info2, sn1, 11), false, false); // should not update + assertEquals(sn3, list.getRegionLocation(2).getServerName()); + assertEquals(sn3, list.getRegionLocation(9).getServerName()); + list = list.updateLocation(hrl(info2, sn1, 11), true, false); // should update + assertEquals(sn1, list.getRegionLocation(2).getServerName()); + assertEquals(sn3, list.getRegionLocation(9).getServerName()); + + // test force update + list = list.updateLocation(hrl(info2, sn2, 9), false, true); // should update + assertEquals(sn2, list.getRegionLocation(2).getServerName()); + assertEquals(sn3, list.getRegionLocation(9).getServerName()); + } + + @Test + public void testMergeLocations() { + RegionLocations list1, list2; + + // test merge empty lists + list1 = new RegionLocations(); + list2 = new RegionLocations(); + + assertTrue(list1 == list1.mergeLocations(list2)); + + // test merge non-empty and empty + list2 = hrll(hrl(info0, sn0)); + list1 = list1.mergeLocations(list2); + assertEquals(sn0, list1.getRegionLocation(0).getServerName()); + + // test merge empty and non empty + list1 = hrll(); + list1 = list2.mergeLocations(list1); + assertEquals(sn0, list1.getRegionLocation(0).getServerName()); + + // test merge non intersecting + list1 = hrll(hrl(info0, sn0), hrl(info1, sn1)); + list2 = hrll(hrl(info2, sn2)); + list1 = list2.mergeLocations(list1); + assertEquals(sn0, list1.getRegionLocation(0).getServerName()); + assertEquals(sn1, list1.getRegionLocation(1).getServerName()); + assertEquals(sn2, list1.getRegionLocation(2).getServerName()); + + // do the other way merge as well + list1 = hrll(hrl(info0, sn0), hrl(info1, sn1)); + list2 = hrll(hrl(info2, sn2)); + list1 = list1.mergeLocations(list2); + assertEquals(sn0, list1.getRegionLocation(0).getServerName()); + assertEquals(sn1, list1.getRegionLocation(1).getServerName()); + assertEquals(sn2, list1.getRegionLocation(2).getServerName()); + + // test intersecting lists same seqNum + list1 = hrll(hrl(info0, sn0), hrl(info1, sn1)); + list2 = hrll(hrl(info0, sn2), hrl(info1, sn2), hrl(info9, sn3)); + list1 = list2.mergeLocations(list1); // list1 should override + assertEquals(10, list1.size()); + assertEquals(sn0, list1.getRegionLocation(0).getServerName()); + assertEquals(sn1, list1.getRegionLocation(1).getServerName()); + assertEquals(sn3, list1.getRegionLocation(9).getServerName()); + + // do the other way + list1 = hrll(hrl(info0, sn0), hrl(info1, sn1)); + list2 = hrll(hrl(info0, sn2), hrl(info1, sn2), hrl(info9, sn3)); + list1 = list1.mergeLocations(list2); // list2 should override + assertEquals(10, list1.size()); + assertEquals(sn2, list1.getRegionLocation(0).getServerName()); + assertEquals(sn2, list1.getRegionLocation(1).getServerName()); + assertEquals(sn3, list1.getRegionLocation(9).getServerName()); + + // test intersecting lists different seqNum + list1 = hrll(hrl(info0, sn0, 10), hrl(info1, sn1, 10)); + list2 = hrll(hrl(info0, sn2, 11), hrl(info1, sn2, 11), hrl(info9, sn3, 11)); + list1 = list1.mergeLocations(list2); // list2 should override because of seqNum + assertEquals(10, list1.size()); + assertEquals(sn2, list1.getRegionLocation(0).getServerName()); + assertEquals(sn2, list1.getRegionLocation(1).getServerName()); + assertEquals(sn3, list1.getRegionLocation(9).getServerName()); + + // do the other way + list1 = hrll(hrl(info0, sn0, 10), hrl(info1, sn1, 10)); + list2 = hrll(hrl(info0, sn2, 11), hrl(info1, sn2, 11), hrl(info9, sn3, 11)); + list1 = list1.mergeLocations(list2); // list2 should override + assertEquals(10, list1.size()); + assertEquals(sn2, list1.getRegionLocation(0).getServerName()); + assertEquals(sn2, list1.getRegionLocation(1).getServerName()); + assertEquals(sn3, list1.getRegionLocation(9).getServerName()); + } +} Modified: hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java?rev=1565041&r1=1565040&r2=1565041&view=diff ============================================================================== --- hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java (original) +++ hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java Thu Feb 6 02:04:53 2014 @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HBaseConf import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.ServerName; @@ -117,8 +118,9 @@ public class TestClientNoCluster extends } @Override - public HRegionLocation getMetaRegionLocation() throws IOException { - return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, META_HOST); + public RegionLocations getMetaRegionLocation() throws IOException { + return new RegionLocations( + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, META_HOST)); } @Override @@ -142,7 +144,7 @@ public class TestClientNoCluster extends * Remove the @Ignore to try out timeout and retry asettings * @throws IOException */ - @Ignore + @Ignore @Test public void testTimeoutAndRetries() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); @@ -759,7 +761,7 @@ public class TestClientNoCluster extends // an exception is thrown -- usually RegionTooBusyException when we have more than // hbase.test.multi.too.many requests outstanding at any time. getConf().setInt("hbase.client.start.log.errors.counter", 0); - + // Ugly but this is only way to pass in configs.into ManyServersManyRegionsConnection class. getConf().setInt("hbase.test.regions", regions); getConf().setLong("hbase.test.namespace.span", namespaceSpan);