From commits-return-65148-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Thu Jan 11 16:31:02 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id BDB43180656 for ; Thu, 11 Jan 2018 16:31:01 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id AC98F160C20; Thu, 11 Jan 2018 15:31:01 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2328B160C44 for ; Thu, 11 Jan 2018 16:30:58 +0100 (CET) Received: (qmail 2936 invoked by uid 500); 11 Jan 2018 15:30:56 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 1983 invoked by uid 99); 11 Jan 2018 15:30:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Jan 2018 15:30:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E94F4F351A; Thu, 11 Jan 2018 15:30:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Thu, 11 Jan 2018 15:31:13 -0000 Message-Id: <86985d8b2cf449bf9fa8f13c177e9ba6@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [24/51] [partial] hbase-site git commit: Published site at . http://git-wip-us.apache.org/repos/asf/hbase-site/blob/f183e80f/devapidocs/src-html/org/apache/hadoop/hbase/wal/RegionGroupingProvider.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/RegionGroupingProvider.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/RegionGroupingProvider.html index 4fec5eb..63c9ca7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/RegionGroupingProvider.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/RegionGroupingProvider.html @@ -35,254 +35,262 @@ 027import java.util.List; 028import java.util.concurrent.ConcurrentHashMap; 029import java.util.concurrent.ConcurrentMap; -030 +030import java.util.concurrent.locks.Lock; 031import org.apache.hadoop.conf.Configuration; -032import org.apache.yetus.audience.InterfaceAudience; -033import org.slf4j.Logger; -034import org.slf4j.LoggerFactory; -035// imports for classes still in regionserver.wal -036import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; -037import org.apache.hadoop.hbase.util.Bytes; -038import org.apache.hadoop.hbase.util.IdLock; -039 -040/** -041 * A WAL Provider that returns a WAL per group of regions. -042 * -043 * This provider follows the decorator pattern and mainly holds the logic for WAL grouping. -044 * WAL creation/roll/close is delegated to {@link #DELEGATE_PROVIDER} -045 * -046 * Region grouping is handled via {@link RegionGroupingStrategy} and can be configured via the -047 * property "hbase.wal.regiongrouping.strategy". Current strategy choices are -048 * <ul> -049 * <li><em>defaultStrategy</em> : Whatever strategy this version of HBase picks. currently -050 * "bounded".</li> -051 * <li><em>identity</em> : each region belongs to its own group.</li> -052 * <li><em>bounded</em> : bounded number of groups and region evenly assigned to each group.</li> -053 * </ul> -054 * Optionally, a FQCN to a custom implementation may be given. -055 */ -056@InterfaceAudience.Private -057public class RegionGroupingProvider implements WALProvider { -058 private static final Logger LOG = LoggerFactory.getLogger(RegionGroupingProvider.class); -059 -060 /** -061 * Map identifiers to a group number. -062 */ -063 public static interface RegionGroupingStrategy { -064 String GROUP_NAME_DELIMITER = "."; -065 -066 /** -067 * Given an identifier and a namespace, pick a group. -068 */ -069 String group(final byte[] identifier, byte[] namespace); -070 void init(Configuration config, String providerId); -071 } -072 -073 /** -074 * Maps between configuration names for strategies and implementation classes. -075 */ -076 static enum Strategies { -077 defaultStrategy(BoundedGroupingStrategy.class), -078 identity(IdentityGroupingStrategy.class), -079 bounded(BoundedGroupingStrategy.class), -080 namespace(NamespaceGroupingStrategy.class); -081 -082 final Class<? extends RegionGroupingStrategy> clazz; -083 Strategies(Class<? extends RegionGroupingStrategy> clazz) { -084 this.clazz = clazz; -085 } -086 } -087 -088 /** -089 * instantiate a strategy from a config property. -090 * requires conf to have already been set (as well as anything the provider might need to read). -091 */ -092 RegionGroupingStrategy getStrategy(final Configuration conf, final String key, -093 final String defaultValue) throws IOException { -094 Class<? extends RegionGroupingStrategy> clazz; -095 try { -096 clazz = Strategies.valueOf(conf.get(key, defaultValue)).clazz; -097 } catch (IllegalArgumentException exception) { -098 // Fall back to them specifying a class name -099 // Note that the passed default class shouldn't actually be used, since the above only fails -100 // when there is a config value present. -101 clazz = conf.getClass(key, IdentityGroupingStrategy.class, RegionGroupingStrategy.class); -102 } -103 LOG.info("Instantiating RegionGroupingStrategy of type " + clazz); -104 try { -105 final RegionGroupingStrategy result = clazz.newInstance(); -106 result.init(conf, providerId); -107 return result; -108 } catch (InstantiationException exception) { -109 LOG.error("couldn't set up region grouping strategy, check config key " + -110 REGION_GROUPING_STRATEGY); -111 LOG.debug("Exception details for failure to load region grouping strategy.", exception); -112 throw new IOException("couldn't set up region grouping strategy", exception); -113 } catch (IllegalAccessException exception) { -114 LOG.error("couldn't set up region grouping strategy, check config key " + -115 REGION_GROUPING_STRATEGY); -116 LOG.debug("Exception details for failure to load region grouping strategy.", exception); -117 throw new IOException("couldn't set up region grouping strategy", exception); -118 } -119 } -120 -121 public static final String REGION_GROUPING_STRATEGY = "hbase.wal.regiongrouping.strategy"; -122 public static final String DEFAULT_REGION_GROUPING_STRATEGY = Strategies.defaultStrategy.name(); -123 -124 /** delegate provider for WAL creation/roll/close */ -125 public static final String DELEGATE_PROVIDER = "hbase.wal.regiongrouping.delegate.provider"; -126 public static final String DEFAULT_DELEGATE_PROVIDER = WALFactory.Providers.defaultProvider -127 .name(); -128 -129 private static final String META_WAL_GROUP_NAME = "meta"; +032import org.apache.hadoop.hbase.HConstants; +033import org.apache.hadoop.hbase.client.RegionInfo; +034// imports for classes still in regionserver.wal +035import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +036import org.apache.hadoop.hbase.util.Bytes; +037import org.apache.hadoop.hbase.util.KeyLocker; +038import org.apache.yetus.audience.InterfaceAudience; +039import org.slf4j.Logger; +040import org.slf4j.LoggerFactory; +041 +042/** +043 * A WAL Provider that returns a WAL per group of regions. +044 * +045 * This provider follows the decorator pattern and mainly holds the logic for WAL grouping. +046 * WAL creation/roll/close is delegated to {@link #DELEGATE_PROVIDER} +047 * +048 * Region grouping is handled via {@link RegionGroupingStrategy} and can be configured via the +049 * property "hbase.wal.regiongrouping.strategy". Current strategy choices are +050 * <ul> +051 * <li><em>defaultStrategy</em> : Whatever strategy this version of HBase picks. currently +052 * "bounded".</li> +053 * <li><em>identity</em> : each region belongs to its own group.</li> +054 * <li><em>bounded</em> : bounded number of groups and region evenly assigned to each group.</li> +055 * </ul> +056 * Optionally, a FQCN to a custom implementation may be given. +057 */ +058@InterfaceAudience.Private +059public class RegionGroupingProvider implements WALProvider { +060 private static final Logger LOG = LoggerFactory.getLogger(RegionGroupingProvider.class); +061 +062 /** +063 * Map identifiers to a group number. +064 */ +065 public static interface RegionGroupingStrategy { +066 String GROUP_NAME_DELIMITER = "."; +067 +068 /** +069 * Given an identifier and a namespace, pick a group. +070 */ +071 String group(final byte[] identifier, byte[] namespace); +072 void init(Configuration config, String providerId); +073 } +074 +075 /** +076 * Maps between configuration names for strategies and implementation classes. +077 */ +078 static enum Strategies { +079 defaultStrategy(BoundedGroupingStrategy.class), +080 identity(IdentityGroupingStrategy.class), +081 bounded(BoundedGroupingStrategy.class), +082 namespace(NamespaceGroupingStrategy.class); +083 +084 final Class<? extends RegionGroupingStrategy> clazz; +085 Strategies(Class<? extends RegionGroupingStrategy> clazz) { +086 this.clazz = clazz; +087 } +088 } +089 +090 /** +091 * instantiate a strategy from a config property. +092 * requires conf to have already been set (as well as anything the provider might need to read). +093 */ +094 RegionGroupingStrategy getStrategy(final Configuration conf, final String key, +095 final String defaultValue) throws IOException { +096 Class<? extends RegionGroupingStrategy> clazz; +097 try { +098 clazz = Strategies.valueOf(conf.get(key, defaultValue)).clazz; +099 } catch (IllegalArgumentException exception) { +100 // Fall back to them specifying a class name +101 // Note that the passed default class shouldn't actually be used, since the above only fails +102 // when there is a config value present. +103 clazz = conf.getClass(key, IdentityGroupingStrategy.class, RegionGroupingStrategy.class); +104 } +105 LOG.info("Instantiating RegionGroupingStrategy of type " + clazz); +106 try { +107 final RegionGroupingStrategy result = clazz.newInstance(); +108 result.init(conf, providerId); +109 return result; +110 } catch (InstantiationException exception) { +111 LOG.error("couldn't set up region grouping strategy, check config key " + +112 REGION_GROUPING_STRATEGY); +113 LOG.debug("Exception details for failure to load region grouping strategy.", exception); +114 throw new IOException("couldn't set up region grouping strategy", exception); +115 } catch (IllegalAccessException exception) { +116 LOG.error("couldn't set up region grouping strategy, check config key " + +117 REGION_GROUPING_STRATEGY); +118 LOG.debug("Exception details for failure to load region grouping strategy.", exception); +119 throw new IOException("couldn't set up region grouping strategy", exception); +120 } +121 } +122 +123 public static final String REGION_GROUPING_STRATEGY = "hbase.wal.regiongrouping.strategy"; +124 public static final String DEFAULT_REGION_GROUPING_STRATEGY = Strategies.defaultStrategy.name(); +125 +126 /** delegate provider for WAL creation/roll/close */ +127 public static final String DELEGATE_PROVIDER = "hbase.wal.regiongrouping.delegate.provider"; +128 public static final String DEFAULT_DELEGATE_PROVIDER = WALFactory.Providers.defaultProvider +129 .name(); 130 -131 /** A group-provider mapping, make sure one-one rather than many-one mapping */ -132 private final ConcurrentMap<String, WALProvider> cached = new ConcurrentHashMap<>(); -133 -134 private final IdLock createLock = new IdLock(); +131 private static final String META_WAL_GROUP_NAME = "meta"; +132 +133 /** A group-provider mapping, make sure one-one rather than many-one mapping */ +134 private final ConcurrentMap<String, WALProvider> cached = new ConcurrentHashMap<>(); 135 -136 private RegionGroupingStrategy strategy = null; -137 private WALFactory factory = null; -138 private List<WALActionsListener> listeners = null; -139 private String providerId = null; -140 private Class<? extends WALProvider> providerClass; -141 -142 @Override -143 public void init(final WALFactory factory, final Configuration conf, -144 final List<WALActionsListener> listeners, final String providerId) throws IOException { -145 if (null != strategy) { -146 throw new IllegalStateException("WALProvider.init should only be called once."); -147 } -148 this.factory = factory; -149 this.listeners = null == listeners ? null : Collections.unmodifiableList(listeners); -150 StringBuilder sb = new StringBuilder().append(factory.factoryId); -151 if (providerId != null) { -152 if (providerId.startsWith(WAL_FILE_NAME_DELIMITER)) { -153 sb.append(providerId); -154 } else { -155 sb.append(WAL_FILE_NAME_DELIMITER).append(providerId); -156 } -157 } -158 this.providerId = sb.toString(); -159 this.strategy = getStrategy(conf, REGION_GROUPING_STRATEGY, DEFAULT_REGION_GROUPING_STRATEGY); -160 this.providerClass = factory.getProviderClass(DELEGATE_PROVIDER, DEFAULT_DELEGATE_PROVIDER); -161 } -162 -163 private WALProvider createProvider(String group) throws IOException { -164 if (META_WAL_PROVIDER_ID.equals(providerId)) { -165 return factory.createProvider(providerClass, listeners, META_WAL_PROVIDER_ID); -166 } else { -167 return factory.createProvider(providerClass, listeners, group); -168 } -169 } -170 -171 @Override -172 public List<WAL> getWALs() { -173 List<WAL> wals = new ArrayList<>(); -174 for (WALProvider provider : cached.values()) { -175 wals.addAll(provider.getWALs()); -176 } -177 return wals; -178 } -179 -180 private WAL getWAL(final String group) throws IOException { -181 WALProvider provider = cached.get(group); -182 if (provider == null) { -183 IdLock.Entry lockEntry = null; -184 try { -185 lockEntry = createLock.getLockEntry(group.hashCode()); -186 provider = cached.get(group); -187 if (provider == null) { -188 provider = createProvider(group); -189 cached.put(group, provider); -190 } -191 } finally { -192 if (lockEntry != null) { -193 createLock.releaseLockEntry(lockEntry); -194 } -195 } -196 } -197 return provider.getWAL(null, null); -198 } -199 -200 @Override -201 public WAL getWAL(final byte[] identifier, byte[] namespace) throws IOException { -202 final String group; -203 if (META_WAL_PROVIDER_ID.equals(this.providerId)) { -204 group = META_WAL_GROUP_NAME; -205 } else { -206 group = strategy.group(identifier, namespace); -207 } -208 return getWAL(group); -209 } -210 -211 @Override -212 public void shutdown() throws IOException { -213 // save the last exception and rethrow -214 IOException failure = null; -215 for (WALProvider provider: cached.values()) { -216 try { -217 provider.shutdown(); -218 } catch (IOException e) { -219 LOG.error("Problem shutting down wal provider '" + provider + "': " + e.getMessage()); -220 if (LOG.isDebugEnabled()) { -221 LOG.debug("Details of problem shutting down wal provider '" + provider + "'", e); -222 } -223 failure = e; -224 } -225 } -226 if (failure != null) { -227 throw failure; -228 } -229 } -230 -231 @Override -232 public void close() throws IOException { -233 // save the last exception and rethrow -234 IOException failure = null; -235 for (WALProvider provider : cached.values()) { -236 try { -237 provider.close(); -238 } catch (IOException e) { -239 LOG.error("Problem closing wal provider '" + provider + "': " + e.getMessage()); -240 if (LOG.isDebugEnabled()) { -241 LOG.debug("Details of problem closing wal provider '" + provider + "'", e); -242 } -243 failure = e; -244 } -245 } -246 if (failure != null) { -247 throw failure; -248 } -249 } -250 -251 static class IdentityGroupingStrategy implements RegionGroupingStrategy { -252 @Override -253 public void init(Configuration config, String providerId) {} -254 @Override -255 public String group(final byte[] identifier, final byte[] namespace) { -256 return Bytes.toString(identifier); -257 } -258 } -259 -260 @Override -261 public long getNumLogFiles() { -262 long numLogFiles = 0; -263 for (WALProvider provider : cached.values()) { -264 numLogFiles += provider.getNumLogFiles(); +136 private final KeyLocker<String> createLock = new KeyLocker<>(); +137 +138 private RegionGroupingStrategy strategy = null; +139 private WALFactory factory = null; +140 private List<WALActionsListener> listeners = null; +141 private String providerId = null; +142 private Class<? extends WALProvider> providerClass; +143 +144 @Override +145 public void init(final WALFactory factory, final Configuration conf, +146 final List<WALActionsListener> listeners, final String providerId) throws IOException { +147 if (null != strategy) { +148 throw new IllegalStateException("WALProvider.init should only be called once."); +149 } +150 this.factory = factory; +151 this.listeners = null == listeners ? null : Collections.unmodifiableList(listeners); +152 StringBuilder sb = new StringBuilder().append(factory.factoryId); +153 if (providerId != null) { +154 if (providerId.startsWith(WAL_FILE_NAME_DELIMITER)) { +155 sb.append(providerId); +156 } else { +157 sb.append(WAL_FILE_NAME_DELIMITER).append(providerId); +158 } +159 } +160 this.providerId = sb.toString(); +161 this.strategy = getStrategy(conf, REGION_GROUPING_STRATEGY, DEFAULT_REGION_GROUPING_STRATEGY); +162 this.providerClass = factory.getProviderClass(DELEGATE_PROVIDER, DEFAULT_DELEGATE_PROVIDER); +163 } +164 +165 private WALProvider createProvider(String group) throws IOException { +166 if (META_WAL_PROVIDER_ID.equals(providerId)) { +167 return factory.createProvider(providerClass, listeners, META_WAL_PROVIDER_ID); +168 } else { +169 return factory.createProvider(providerClass, listeners, group); +170 } +171 } +172 +173 @Override +174 public List<WAL> getWALs() { +175 List<WAL> wals = new ArrayList<>(); +176 for (WALProvider provider : cached.values()) { +177 wals.addAll(provider.getWALs()); +178 } +179 return wals; +180 } +181 +182 private WAL getWAL(String group) throws IOException { +183 WALProvider provider = cached.get(group); +184 if (provider == null) { +185 Lock lock = createLock.acquireLock(group); +186 try { +187 provider = cached.get(group); +188 if (provider == null) { +189 provider = createProvider(group); +190 cached.put(group, provider); +191 } +192 } finally { +193 lock.unlock(); +194 } +195 } +196 return provider.getWAL(null); +197 } +198 +199 @Override +200 public WAL getWAL(RegionInfo region) throws IOException { +201 String group; +202 if (META_WAL_PROVIDER_ID.equals(this.providerId)) { +203 group = META_WAL_GROUP_NAME; +204 } else { +205 byte[] id; +206 byte[] namespace; +207 if (region != null) { +208 id = region.getEncodedNameAsBytes(); +209 namespace = region.getTable().getNamespace(); +210 } else { +211 id = HConstants.EMPTY_BYTE_ARRAY; +212 namespace = null; +213 } +214 group = strategy.group(id, namespace); +215 } +216 return getWAL(group); +217 } +218 +219 @Override +220 public void shutdown() throws IOException { +221 // save the last exception and rethrow +222 IOException failure = null; +223 for (WALProvider provider: cached.values()) { +224 try { +225 provider.shutdown(); +226 } catch (IOException e) { +227 LOG.error("Problem shutting down wal provider '" + provider + "': " + e.getMessage()); +228 if (LOG.isDebugEnabled()) { +229 LOG.debug("Details of problem shutting down wal provider '" + provider + "'", e); +230 } +231 failure = e; +232 } +233 } +234 if (failure != null) { +235 throw failure; +236 } +237 } +238 +239 @Override +240 public void close() throws IOException { +241 // save the last exception and rethrow +242 IOException failure = null; +243 for (WALProvider provider : cached.values()) { +244 try { +245 provider.close(); +246 } catch (IOException e) { +247 LOG.error("Problem closing wal provider '" + provider + "': " + e.getMessage()); +248 if (LOG.isDebugEnabled()) { +249 LOG.debug("Details of problem closing wal provider '" + provider + "'", e); +250 } +251 failure = e; +252 } +253 } +254 if (failure != null) { +255 throw failure; +256 } +257 } +258 +259 static class IdentityGroupingStrategy implements RegionGroupingStrategy { +260 @Override +261 public void init(Configuration config, String providerId) {} +262 @Override +263 public String group(final byte[] identifier, final byte[] namespace) { +264 return Bytes.toString(identifier); 265 } -266 return numLogFiles; -267 } -268 -269 @Override -270 public long getLogFileSize() { -271 long logFileSize = 0; -272 for (WALProvider provider : cached.values()) { -273 logFileSize += provider.getLogFileSize(); -274 } -275 return logFileSize; -276 } -277} +266 } +267 +268 @Override +269 public long getNumLogFiles() { +270 long numLogFiles = 0; +271 for (WALProvider provider : cached.values()) { +272 numLogFiles += provider.getNumLogFiles(); +273 } +274 return numLogFiles; +275 } +276 +277 @Override +278 public long getLogFileSize() { +279 long logFileSize = 0; +280 for (WALProvider provider : cached.values()) { +281 logFileSize += provider.getLogFileSize(); +282 } +283 return logFileSize; +284 } +285}