Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EEB77195FC for ; Thu, 24 Mar 2016 15:54:48 +0000 (UTC) Received: (qmail 70764 invoked by uid 500); 24 Mar 2016 15:54:44 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 70670 invoked by uid 500); 24 Mar 2016 15:54:44 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 67473 invoked by uid 99); 24 Mar 2016 15:54:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 24 Mar 2016 15:54:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E0E89DFDCF; Thu, 24 Mar 2016 15:54:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Thu, 24 Mar 2016 15:55:11 -0000 Message-Id: In-Reply-To: <4b92113c539c459a9ed56e402685cba1@git.apache.org> References: <4b92113c539c459a9ed56e402685cba1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [31/51] [partial] hbase-site git commit: Published site at 52fd70500e0a00e273e2ec0c09d7c914b89432ce. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/f30982bd/devapidocs/src-html/org/apache/hadoop/hbase/client/MetricsConnection.NewMetric.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/client/MetricsConnection.NewMetric.html b/devapidocs/src-html/org/apache/hadoop/hbase/client/MetricsConnection.NewMetric.html index 9e42ac9..054df6f 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/client/MetricsConnection.NewMetric.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/client/MetricsConnection.NewMetric.html @@ -71,411 +71,413 @@ 063 private static final String MEMLOAD_BASE = "memstoreLoad_"; 064 private static final String HEAP_BASE = "heapOccupancy_"; 065 private static final String CACHE_BASE = "cacheDroppingExceptions_"; -066 private static final String CLIENT_SVC = ClientService.getDescriptor().getName(); -067 -068 /** A container class for collecting details about the RPC call as it percolates. */ -069 public static class CallStats { -070 private long requestSizeBytes = 0; -071 private long responseSizeBytes = 0; -072 private long startTime = 0; -073 private long callTimeMs = 0; -074 -075 public long getRequestSizeBytes() { -076 return requestSizeBytes; -077 } -078 -079 public void setRequestSizeBytes(long requestSizeBytes) { -080 this.requestSizeBytes = requestSizeBytes; -081 } -082 -083 public long getResponseSizeBytes() { -084 return responseSizeBytes; -085 } -086 -087 public void setResponseSizeBytes(long responseSizeBytes) { -088 this.responseSizeBytes = responseSizeBytes; -089 } -090 -091 public long getStartTime() { -092 return startTime; -093 } -094 -095 public void setStartTime(long startTime) { -096 this.startTime = startTime; -097 } -098 -099 public long getCallTimeMs() { -100 return callTimeMs; -101 } -102 -103 public void setCallTimeMs(long callTimeMs) { -104 this.callTimeMs = callTimeMs; -105 } -106 } -107 -108 @VisibleForTesting -109 protected static final class CallTracker { -110 private final String name; -111 @VisibleForTesting final Timer callTimer; -112 @VisibleForTesting final Histogram reqHist; -113 @VisibleForTesting final Histogram respHist; -114 -115 private CallTracker(MetricRegistry registry, String name, String subName, String scope) { -116 StringBuilder sb = new StringBuilder(CLIENT_SVC).append("_").append(name); -117 if (subName != null) { -118 sb.append("(").append(subName).append(")"); -119 } -120 this.name = sb.toString(); -121 this.callTimer = registry.timer(name(MetricsConnection.class, -122 DRTN_BASE + this.name, scope)); -123 this.reqHist = registry.histogram(name(MetricsConnection.class, -124 REQ_BASE + this.name, scope)); -125 this.respHist = registry.histogram(name(MetricsConnection.class, -126 RESP_BASE + this.name, scope)); -127 } -128 -129 private CallTracker(MetricRegistry registry, String name, String scope) { -130 this(registry, name, null, scope); -131 } -132 -133 public void updateRpc(CallStats stats) { -134 this.callTimer.update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS); -135 this.reqHist.update(stats.getRequestSizeBytes()); -136 this.respHist.update(stats.getResponseSizeBytes()); -137 } -138 -139 @Override -140 public String toString() { -141 return "CallTracker:" + name; -142 } -143 } -144 -145 protected static class RegionStats { -146 final String name; -147 final Histogram memstoreLoadHist; -148 final Histogram heapOccupancyHist; -149 -150 public RegionStats(MetricRegistry registry, String name) { -151 this.name = name; -152 this.memstoreLoadHist = registry.histogram(name(MetricsConnection.class, -153 MEMLOAD_BASE + this.name)); -154 this.heapOccupancyHist = registry.histogram(name(MetricsConnection.class, -155 HEAP_BASE + this.name)); -156 } -157 -158 public void update(ClientProtos.RegionLoadStats regionStatistics) { -159 this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad()); -160 this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy()); -161 } -162 } -163 -164 @VisibleForTesting -165 protected static class RunnerStats { -166 final Counter normalRunners; -167 final Counter delayRunners; -168 final Histogram delayIntevalHist; -169 -170 public RunnerStats(MetricRegistry registry) { -171 this.normalRunners = registry.counter( -172 name(MetricsConnection.class, "normalRunnersCount")); -173 this.delayRunners = registry.counter( -174 name(MetricsConnection.class, "delayRunnersCount")); -175 this.delayIntevalHist = registry.histogram( -176 name(MetricsConnection.class, "delayIntervalHist")); -177 } -178 -179 public void incrNormalRunners() { -180 this.normalRunners.inc(); -181 } -182 -183 public void incrDelayRunners() { -184 this.delayRunners.inc(); -185 } -186 -187 public void updateDelayInterval(long interval) { -188 this.delayIntevalHist.update(interval); -189 } -190 } -191 -192 @VisibleForTesting -193 protected ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats -194 = new ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>>(); -195 -196 public void updateServerStats(ServerName serverName, byte[] regionName, -197 Object r) { -198 if (!(r instanceof Result)) { -199 return; -200 } -201 Result result = (Result) r; -202 ClientProtos.RegionLoadStats stats = result.getStats(); -203 if (stats == null) { -204 return; -205 } -206 updateRegionStats(serverName, regionName, stats); -207 } -208 -209 @Override -210 public void updateRegionStats(ServerName serverName, byte[] regionName, -211 ClientProtos.RegionLoadStats stats) { -212 String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName); -213 ConcurrentMap<byte[], RegionStats> rsStats = null; -214 if (serverStats.containsKey(serverName)) { -215 rsStats = serverStats.get(serverName); -216 } else { -217 rsStats = serverStats.putIfAbsent(serverName, -218 new ConcurrentSkipListMap<byte[], RegionStats>(Bytes.BYTES_COMPARATOR)); -219 if (rsStats == null) { -220 rsStats = serverStats.get(serverName); -221 } -222 } -223 RegionStats regionStats = null; -224 if (rsStats.containsKey(regionName)) { -225 regionStats = rsStats.get(regionName); -226 } else { -227 regionStats = rsStats.putIfAbsent(regionName, new RegionStats(this.registry, name)); -228 if (regionStats == null) { -229 regionStats = rsStats.get(regionName); -230 } -231 } -232 regionStats.update(stats); -233 } -234 +066 private static final String UNKNOWN_EXCEPTION = "UnknownException"; +067 private static final String CLIENT_SVC = ClientService.getDescriptor().getName(); +068 +069 /** A container class for collecting details about the RPC call as it percolates. */ +070 public static class CallStats { +071 private long requestSizeBytes = 0; +072 private long responseSizeBytes = 0; +073 private long startTime = 0; +074 private long callTimeMs = 0; +075 +076 public long getRequestSizeBytes() { +077 return requestSizeBytes; +078 } +079 +080 public void setRequestSizeBytes(long requestSizeBytes) { +081 this.requestSizeBytes = requestSizeBytes; +082 } +083 +084 public long getResponseSizeBytes() { +085 return responseSizeBytes; +086 } +087 +088 public void setResponseSizeBytes(long responseSizeBytes) { +089 this.responseSizeBytes = responseSizeBytes; +090 } +091 +092 public long getStartTime() { +093 return startTime; +094 } +095 +096 public void setStartTime(long startTime) { +097 this.startTime = startTime; +098 } +099 +100 public long getCallTimeMs() { +101 return callTimeMs; +102 } +103 +104 public void setCallTimeMs(long callTimeMs) { +105 this.callTimeMs = callTimeMs; +106 } +107 } +108 +109 @VisibleForTesting +110 protected static final class CallTracker { +111 private final String name; +112 @VisibleForTesting final Timer callTimer; +113 @VisibleForTesting final Histogram reqHist; +114 @VisibleForTesting final Histogram respHist; +115 +116 private CallTracker(MetricRegistry registry, String name, String subName, String scope) { +117 StringBuilder sb = new StringBuilder(CLIENT_SVC).append("_").append(name); +118 if (subName != null) { +119 sb.append("(").append(subName).append(")"); +120 } +121 this.name = sb.toString(); +122 this.callTimer = registry.timer(name(MetricsConnection.class, +123 DRTN_BASE + this.name, scope)); +124 this.reqHist = registry.histogram(name(MetricsConnection.class, +125 REQ_BASE + this.name, scope)); +126 this.respHist = registry.histogram(name(MetricsConnection.class, +127 RESP_BASE + this.name, scope)); +128 } +129 +130 private CallTracker(MetricRegistry registry, String name, String scope) { +131 this(registry, name, null, scope); +132 } +133 +134 public void updateRpc(CallStats stats) { +135 this.callTimer.update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS); +136 this.reqHist.update(stats.getRequestSizeBytes()); +137 this.respHist.update(stats.getResponseSizeBytes()); +138 } +139 +140 @Override +141 public String toString() { +142 return "CallTracker:" + name; +143 } +144 } +145 +146 protected static class RegionStats { +147 final String name; +148 final Histogram memstoreLoadHist; +149 final Histogram heapOccupancyHist; +150 +151 public RegionStats(MetricRegistry registry, String name) { +152 this.name = name; +153 this.memstoreLoadHist = registry.histogram(name(MetricsConnection.class, +154 MEMLOAD_BASE + this.name)); +155 this.heapOccupancyHist = registry.histogram(name(MetricsConnection.class, +156 HEAP_BASE + this.name)); +157 } +158 +159 public void update(ClientProtos.RegionLoadStats regionStatistics) { +160 this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad()); +161 this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy()); +162 } +163 } +164 +165 @VisibleForTesting +166 protected static class RunnerStats { +167 final Counter normalRunners; +168 final Counter delayRunners; +169 final Histogram delayIntevalHist; +170 +171 public RunnerStats(MetricRegistry registry) { +172 this.normalRunners = registry.counter( +173 name(MetricsConnection.class, "normalRunnersCount")); +174 this.delayRunners = registry.counter( +175 name(MetricsConnection.class, "delayRunnersCount")); +176 this.delayIntevalHist = registry.histogram( +177 name(MetricsConnection.class, "delayIntervalHist")); +178 } +179 +180 public void incrNormalRunners() { +181 this.normalRunners.inc(); +182 } +183 +184 public void incrDelayRunners() { +185 this.delayRunners.inc(); +186 } +187 +188 public void updateDelayInterval(long interval) { +189 this.delayIntevalHist.update(interval); +190 } +191 } +192 +193 @VisibleForTesting +194 protected ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats +195 = new ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>>(); +196 +197 public void updateServerStats(ServerName serverName, byte[] regionName, +198 Object r) { +199 if (!(r instanceof Result)) { +200 return; +201 } +202 Result result = (Result) r; +203 ClientProtos.RegionLoadStats stats = result.getStats(); +204 if (stats == null) { +205 return; +206 } +207 updateRegionStats(serverName, regionName, stats); +208 } +209 +210 @Override +211 public void updateRegionStats(ServerName serverName, byte[] regionName, +212 ClientProtos.RegionLoadStats stats) { +213 String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName); +214 ConcurrentMap<byte[], RegionStats> rsStats = null; +215 if (serverStats.containsKey(serverName)) { +216 rsStats = serverStats.get(serverName); +217 } else { +218 rsStats = serverStats.putIfAbsent(serverName, +219 new ConcurrentSkipListMap<byte[], RegionStats>(Bytes.BYTES_COMPARATOR)); +220 if (rsStats == null) { +221 rsStats = serverStats.get(serverName); +222 } +223 } +224 RegionStats regionStats = null; +225 if (rsStats.containsKey(regionName)) { +226 regionStats = rsStats.get(regionName); +227 } else { +228 regionStats = rsStats.putIfAbsent(regionName, new RegionStats(this.registry, name)); +229 if (regionStats == null) { +230 regionStats = rsStats.get(regionName); +231 } +232 } +233 regionStats.update(stats); +234 } 235 -236 /** A lambda for dispatching to the appropriate metric factory method */ -237 private static interface NewMetric<T> { -238 T newMetric(Class<?> clazz, String name, String scope); -239 } -240 -241 /** Anticipated number of metric entries */ -242 private static final int CAPACITY = 50; -243 /** Default load factor from {@link java.util.HashMap#DEFAULT_LOAD_FACTOR} */ -244 private static final float LOAD_FACTOR = 0.75f; -245 /** -246 * Anticipated number of concurrent accessor threads, from -247 * {@link ConnectionImplementation#getBatchPool()} -248 */ -249 private static final int CONCURRENCY_LEVEL = 256; -250 -251 private final MetricRegistry registry; -252 private final JmxReporter reporter; -253 private final String scope; -254 -255 private final NewMetric<Timer> timerFactory = new NewMetric<Timer>() { -256 @Override public Timer newMetric(Class<?> clazz, String name, String scope) { -257 return registry.timer(name(clazz, name, scope)); -258 } -259 }; -260 -261 private final NewMetric<Histogram> histogramFactory = new NewMetric<Histogram>() { -262 @Override public Histogram newMetric(Class<?> clazz, String name, String scope) { -263 return registry.histogram(name(clazz, name, scope)); -264 } -265 }; -266 -267 private final NewMetric<Counter> counterFactory = new NewMetric<Counter>() { -268 @Override public Counter newMetric(Class<?> clazz, String name, String scope) { -269 return registry.counter(name(clazz, name, scope)); -270 } -271 }; -272 -273 // static metrics -274 -275 @VisibleForTesting protected final Counter metaCacheHits; -276 @VisibleForTesting protected final Counter metaCacheMisses; -277 @VisibleForTesting protected final CallTracker getTracker; -278 @VisibleForTesting protected final CallTracker scanTracker; -279 @VisibleForTesting protected final CallTracker appendTracker; -280 @VisibleForTesting protected final CallTracker deleteTracker; -281 @VisibleForTesting protected final CallTracker incrementTracker; -282 @VisibleForTesting protected final CallTracker putTracker; -283 @VisibleForTesting protected final CallTracker multiTracker; -284 @VisibleForTesting protected final RunnerStats runnerStats; -285 private final Counter metaCacheNumClearServer; -286 private final Counter metaCacheNumClearRegion; -287 -288 // dynamic metrics -289 -290 // These maps are used to cache references to the metric instances that are managed by the -291 // registry. I don't think their use perfectly removes redundant allocations, but it's -292 // a big improvement over calling registry.newMetric each time. -293 @VisibleForTesting protected final ConcurrentMap<String, Timer> rpcTimers = -294 new ConcurrentHashMap<>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); -295 @VisibleForTesting protected final ConcurrentMap<String, Histogram> rpcHistograms = -296 new ConcurrentHashMap<>(CAPACITY * 2 /* tracking both request and response sizes */, -297 LOAD_FACTOR, CONCURRENCY_LEVEL); -298 private final ConcurrentMap<String, Counter> cacheDroppingExceptions = -299 new ConcurrentHashMap<>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); -300 -301 public MetricsConnection(final ConnectionImplementation conn) { -302 this.scope = conn.toString(); -303 this.registry = new MetricRegistry(); -304 final ThreadPoolExecutor batchPool = (ThreadPoolExecutor) conn.getCurrentBatchPool(); -305 final ThreadPoolExecutor metaPool = (ThreadPoolExecutor) conn.getCurrentMetaLookupPool(); -306 -307 this.registry.register(name(this.getClass(), "executorPoolActiveThreads", scope), -308 new RatioGauge() { -309 @Override -310 protected Ratio getRatio() { -311 return Ratio.of(batchPool.getActiveCount(), batchPool.getMaximumPoolSize()); -312 } -313 }); -314 this.registry.register(name(this.getClass(), "metaPoolActiveThreads", scope), -315 new RatioGauge() { -316 @Override -317 protected Ratio getRatio() { -318 return Ratio.of(metaPool.getActiveCount(), metaPool.getMaximumPoolSize()); -319 } -320 }); -321 this.metaCacheHits = registry.counter(name(this.getClass(), "metaCacheHits", scope)); -322 this.metaCacheMisses = registry.counter(name(this.getClass(), "metaCacheMisses", scope)); -323 this.metaCacheNumClearServer = registry.counter(name(this.getClass(), -324 "metaCacheNumClearServer", scope)); -325 this.metaCacheNumClearRegion = registry.counter(name(this.getClass(), -326 "metaCacheNumClearRegion", scope)); -327 this.getTracker = new CallTracker(this.registry, "Get", scope); -328 this.scanTracker = new CallTracker(this.registry, "Scan", scope); -329 this.appendTracker = new CallTracker(this.registry, "Mutate", "Append", scope); -330 this.deleteTracker = new CallTracker(this.registry, "Mutate", "Delete", scope); -331 this.incrementTracker = new CallTracker(this.registry, "Mutate", "Increment", scope); -332 this.putTracker = new CallTracker(this.registry, "Mutate", "Put", scope); -333 this.multiTracker = new CallTracker(this.registry, "Multi", scope); -334 this.runnerStats = new RunnerStats(this.registry); -335 -336 this.reporter = JmxReporter.forRegistry(this.registry).build(); -337 this.reporter.start(); -338 } -339 -340 public void shutdown() { -341 this.reporter.stop(); -342 } -343 -344 /** Produce an instance of {@link CallStats} for clients to attach to RPCs. */ -345 public static CallStats newCallStats() { -346 // TODO: instance pool to reduce GC? -347 return new CallStats(); -348 } -349 -350 /** Increment the number of meta cache hits. */ -351 public void incrMetaCacheHit() { -352 metaCacheHits.inc(); -353 } -354 -355 /** Increment the number of meta cache misses. */ -356 public void incrMetaCacheMiss() { -357 metaCacheMisses.inc(); -358 } -359 -360 /** Increment the number of meta cache drops requested for entire RegionServer. */ -361 public void incrMetaCacheNumClearServer() { -362 metaCacheNumClearServer.inc(); -363 } -364 -365 /** Increment the number of meta cache drops requested for individual region. */ -366 public void incrMetaCacheNumClearRegion() { -367 metaCacheNumClearRegion.inc(); -368 } -369 -370 /** Increment the number of normal runner counts. */ -371 public void incrNormalRunners() { -372 this.runnerStats.incrNormalRunners(); -373 } -374 -375 /** Increment the number of delay runner counts. */ -376 public void incrDelayRunners() { -377 this.runnerStats.incrDelayRunners(); -378 } -379 -380 /** Update delay interval of delay runner. */ -381 public void updateDelayInterval(long interval) { -382 this.runnerStats.updateDelayInterval(interval); -383 } -384 -385 /** -386 * Get a metric for {@code key} from {@code map}, or create it with {@code factory}. -387 */ -388 private <T> T getMetric(String key, ConcurrentMap<String, T> map, NewMetric<T> factory) { -389 T t = map.get(key); -390 if (t == null) { -391 t = factory.newMetric(this.getClass(), key, scope); -392 T tmp = map.putIfAbsent(key, t); -393 t = (tmp == null) ? t : tmp; -394 } -395 return t; -396 } -397 -398 /** Update call stats for non-critical-path methods */ -399 private void updateRpcGeneric(MethodDescriptor method, CallStats stats) { -400 final String methodName = method.getService().getName() + "_" + method.getName(); -401 getMetric(DRTN_BASE + methodName, rpcTimers, timerFactory) -402 .update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS); -403 getMetric(REQ_BASE + methodName, rpcHistograms, histogramFactory) -404 .update(stats.getRequestSizeBytes()); -405 getMetric(RESP_BASE + methodName, rpcHistograms, histogramFactory) -406 .update(stats.getResponseSizeBytes()); -407 } -408 -409 /** Report RPC context to metrics system. */ -410 public void updateRpc(MethodDescriptor method, Message param, CallStats stats) { -411 // this implementation is tied directly to protobuf implementation details. would be better -412 // if we could dispatch based on something static, ie, request Message type. -413 if (method.getService() == ClientService.getDescriptor()) { -414 switch(method.getIndex()) { -415 case 0: -416 assert "Get".equals(method.getName()); -417 getTracker.updateRpc(stats); -418 return; -419 case 1: -420 assert "Mutate".equals(method.getName()); -421 final MutationType mutationType = ((MutateRequest) param).getMutation().getMutateType(); -422 switch(mutationType) { -423 case APPEND: -424 appendTracker.updateRpc(stats); -425 return; -426 case DELETE: -427 deleteTracker.updateRpc(stats); -428 return; -429 case INCREMENT: -430 incrementTracker.updateRpc(stats); -431 return; -432 case PUT: -433 putTracker.updateRpc(stats); -434 return; -435 default: -436 throw new RuntimeException("Unrecognized mutation type " + mutationType); -437 } -438 case 2: -439 assert "Scan".equals(method.getName()); -440 scanTracker.updateRpc(stats); -441 return; -442 case 3: -443 assert "BulkLoadHFile".equals(method.getName()); -444 // use generic implementation -445 break; -446 case 4: -447 assert "ExecService".equals(method.getName()); -448 // use generic implementation -449 break; -450 case 5: -451 assert "ExecRegionServerService".equals(method.getName()); -452 // use generic implementation -453 break; -454 case 6: -455 assert "Multi".equals(method.getName()); -456 multiTracker.updateRpc(stats); -457 return; -458 default: -459 throw new RuntimeException("Unrecognized ClientService RPC type " + method.getFullName()); -460 } -461 } -462 // Fallback to dynamic registry lookup for DDL methods. -463 updateRpcGeneric(method, stats); -464 } -465 -466 public void incrCacheDroppingExceptions(Object exception) { -467 getMetric(CACHE_BASE + exception.getClass().getSimpleName(), -468 cacheDroppingExceptions, counterFactory).inc(); -469 } -470} +236 +237 /** A lambda for dispatching to the appropriate metric factory method */ +238 private static interface NewMetric<T> { +239 T newMetric(Class<?> clazz, String name, String scope); +240 } +241 +242 /** Anticipated number of metric entries */ +243 private static final int CAPACITY = 50; +244 /** Default load factor from {@link java.util.HashMap#DEFAULT_LOAD_FACTOR} */ +245 private static final float LOAD_FACTOR = 0.75f; +246 /** +247 * Anticipated number of concurrent accessor threads, from +248 * {@link ConnectionImplementation#getBatchPool()} +249 */ +250 private static final int CONCURRENCY_LEVEL = 256; +251 +252 private final MetricRegistry registry; +253 private final JmxReporter reporter; +254 private final String scope; +255 +256 private final NewMetric<Timer> timerFactory = new NewMetric<Timer>() { +257 @Override public Timer newMetric(Class<?> clazz, String name, String scope) { +258 return registry.timer(name(clazz, name, scope)); +259 } +260 }; +261 +262 private final NewMetric<Histogram> histogramFactory = new NewMetric<Histogram>() { +263 @Override public Histogram newMetric(Class<?> clazz, String name, String scope) { +264 return registry.histogram(name(clazz, name, scope)); +265 } +266 }; +267 +268 private final NewMetric<Counter> counterFactory = new NewMetric<Counter>() { +269 @Override public Counter newMetric(Class<?> clazz, String name, String scope) { +270 return registry.counter(name(clazz, name, scope)); +271 } +272 }; +273 +274 // static metrics +275 +276 @VisibleForTesting protected final Counter metaCacheHits; +277 @VisibleForTesting protected final Counter metaCacheMisses; +278 @VisibleForTesting protected final CallTracker getTracker; +279 @VisibleForTesting protected final CallTracker scanTracker; +280 @VisibleForTesting protected final CallTracker appendTracker; +281 @VisibleForTesting protected final CallTracker deleteTracker; +282 @VisibleForTesting protected final CallTracker incrementTracker; +283 @VisibleForTesting protected final CallTracker putTracker; +284 @VisibleForTesting protected final CallTracker multiTracker; +285 @VisibleForTesting protected final RunnerStats runnerStats; +286 private final Counter metaCacheNumClearServer; +287 private final Counter metaCacheNumClearRegion; +288 +289 // dynamic metrics +290 +291 // These maps are used to cache references to the metric instances that are managed by the +292 // registry. I don't think their use perfectly removes redundant allocations, but it's +293 // a big improvement over calling registry.newMetric each time. +294 @VisibleForTesting protected final ConcurrentMap<String, Timer> rpcTimers = +295 new ConcurrentHashMap<>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); +296 @VisibleForTesting protected final ConcurrentMap<String, Histogram> rpcHistograms = +297 new ConcurrentHashMap<>(CAPACITY * 2 /* tracking both request and response sizes */, +298 LOAD_FACTOR, CONCURRENCY_LEVEL); +299 private final ConcurrentMap<String, Counter> cacheDroppingExceptions = +300 new ConcurrentHashMap<>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); +301 +302 public MetricsConnection(final ConnectionImplementation conn) { +303 this.scope = conn.toString(); +304 this.registry = new MetricRegistry(); +305 final ThreadPoolExecutor batchPool = (ThreadPoolExecutor) conn.getCurrentBatchPool(); +306 final ThreadPoolExecutor metaPool = (ThreadPoolExecutor) conn.getCurrentMetaLookupPool(); +307 +308 this.registry.register(name(this.getClass(), "executorPoolActiveThreads", scope), +309 new RatioGauge() { +310 @Override +311 protected Ratio getRatio() { +312 return Ratio.of(batchPool.getActiveCount(), batchPool.getMaximumPoolSize()); +313 } +314 }); +315 this.registry.register(name(this.getClass(), "metaPoolActiveThreads", scope), +316 new RatioGauge() { +317 @Override +318 protected Ratio getRatio() { +319 return Ratio.of(metaPool.getActiveCount(), metaPool.getMaximumPoolSize()); +320 } +321 }); +322 this.metaCacheHits = registry.counter(name(this.getClass(), "metaCacheHits", scope)); +323 this.metaCacheMisses = registry.counter(name(this.getClass(), "metaCacheMisses", scope)); +324 this.metaCacheNumClearServer = registry.counter(name(this.getClass(), +325 "metaCacheNumClearServer", scope)); +326 this.metaCacheNumClearRegion = registry.counter(name(this.getClass(), +327 "metaCacheNumClearRegion", scope)); +328 this.getTracker = new CallTracker(this.registry, "Get", scope); +329 this.scanTracker = new CallTracker(this.registry, "Scan", scope); +330 this.appendTracker = new CallTracker(this.registry, "Mutate", "Append", scope); +331 this.deleteTracker = new CallTracker(this.registry, "Mutate", "Delete", scope); +332 this.incrementTracker = new CallTracker(this.registry, "Mutate", "Increment", scope); +333 this.putTracker = new CallTracker(this.registry, "Mutate", "Put", scope); +334 this.multiTracker = new CallTracker(this.registry, "Multi", scope); +335 this.runnerStats = new RunnerStats(this.registry); +336 +337 this.reporter = JmxReporter.forRegistry(this.registry).build(); +338 this.reporter.start(); +339 } +340 +341 public void shutdown() { +342 this.reporter.stop(); +343 } +344 +345 /** Produce an instance of {@link CallStats} for clients to attach to RPCs. */ +346 public static CallStats newCallStats() { +347 // TODO: instance pool to reduce GC? +348 return new CallStats(); +349 } +350 +351 /** Increment the number of meta cache hits. */ +352 public void incrMetaCacheHit() { +353 metaCacheHits.inc(); +354 } +355 +356 /** Increment the number of meta cache misses. */ +357 public void incrMetaCacheMiss() { +358 metaCacheMisses.inc(); +359 } +360 +361 /** Increment the number of meta cache drops requested for entire RegionServer. */ +362 public void incrMetaCacheNumClearServer() { +363 metaCacheNumClearServer.inc(); +364 } +365 +366 /** Increment the number of meta cache drops requested for individual region. */ +367 public void incrMetaCacheNumClearRegion() { +368 metaCacheNumClearRegion.inc(); +369 } +370 +371 /** Increment the number of normal runner counts. */ +372 public void incrNormalRunners() { +373 this.runnerStats.incrNormalRunners(); +374 } +375 +376 /** Increment the number of delay runner counts. */ +377 public void incrDelayRunners() { +378 this.runnerStats.incrDelayRunners(); +379 } +380 +381 /** Update delay interval of delay runner. */ +382 public void updateDelayInterval(long interval) { +383 this.runnerStats.updateDelayInterval(interval); +384 } +385 +386 /** +387 * Get a metric for {@code key} from {@code map}, or create it with {@code factory}. +388 */ +389 private <T> T getMetric(String key, ConcurrentMap<String, T> map, NewMetric<T> factory) { +390 T t = map.get(key); +391 if (t == null) { +392 t = factory.newMetric(this.getClass(), key, scope); +393 T tmp = map.putIfAbsent(key, t); +394 t = (tmp == null) ? t : tmp; +395 } +396 return t; +397 } +398 +399 /** Update call stats for non-critical-path methods */ +400 private void updateRpcGeneric(MethodDescriptor method, CallStats stats) { +401 final String methodName = method.getService().getName() + "_" + method.getName(); +402 getMetric(DRTN_BASE + methodName, rpcTimers, timerFactory) +403 .update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS); +404 getMetric(REQ_BASE + methodName, rpcHistograms, histogramFactory) +405 .update(stats.getRequestSizeBytes()); +406 getMetric(RESP_BASE + methodName, rpcHistograms, histogramFactory) +407 .update(stats.getResponseSizeBytes()); +408 } +409 +410 /** Report RPC context to metrics system. */ +411 public void updateRpc(MethodDescriptor method, Message param, CallStats stats) { +412 // this implementation is tied directly to protobuf implementation details. would be better +413 // if we could dispatch based on something static, ie, request Message type. +414 if (method.getService() == ClientService.getDescriptor()) { +415 switch(method.getIndex()) { +416 case 0: +417 assert "Get".equals(method.getName()); +418 getTracker.updateRpc(stats); +419 return; +420 case 1: +421 assert "Mutate".equals(method.getName()); +422 final MutationType mutationType = ((MutateRequest) param).getMutation().getMutateType(); +423 switch(mutationType) { +424 case APPEND: +425 appendTracker.updateRpc(stats); +426 return; +427 case DELETE: +428 deleteTracker.updateRpc(stats); +429 return; +430 case INCREMENT: +431 incrementTracker.updateRpc(stats); +432 return; +433 case PUT: +434 putTracker.updateRpc(stats); +435 return; +436 default: +437 throw new RuntimeException("Unrecognized mutation type " + mutationType); +438 } +439 case 2: +440 assert "Scan".equals(method.getName()); +441 scanTracker.updateRpc(stats); +442 return; +443 case 3: +444 assert "BulkLoadHFile".equals(method.getName()); +445 // use generic implementation +446 break; +447 case 4: +448 assert "ExecService".equals(method.getName()); +449 // use generic implementation +450 break; +451 case 5: +452 assert "ExecRegionServerService".equals(method.getName()); +453 // use generic implementation +454 break; +455 case 6: +456 assert "Multi".equals(method.getName()); +457 multiTracker.updateRpc(stats); +458 return; +459 default: +460 throw new RuntimeException("Unrecognized ClientService RPC type " + method.getFullName()); +461 } +462 } +463 // Fallback to dynamic registry lookup for DDL methods. +464 updateRpcGeneric(method, stats); +465 } +466 +467 public void incrCacheDroppingExceptions(Object exception) { +468 getMetric(CACHE_BASE + +469 (exception == null? UNKNOWN_EXCEPTION : exception.getClass().getSimpleName()), +470 cacheDroppingExceptions, counterFactory).inc(); +471 } +472}