hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From git-site-r...@apache.org
Subject [18/28] hbase-site git commit: Published site at .
Date Sat, 20 Jan 2018 15:31:38 GMT
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.CleanerContext.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.CleanerContext.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.CleanerContext.html
index f02bbe3..6d501ca 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.CleanerContext.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.CleanerContext.html
@@ -185,52 +185,60 @@
 <span class="sourceLineNo">177</span>    }<a name="line.177"></a>
 <span class="sourceLineNo">178</span>  }<a name="line.178"></a>
 <span class="sourceLineNo">179</span><a name="line.179"></a>
-<span class="sourceLineNo">180</span>  private static final class CleanerContext {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    // At most waits 60 seconds<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    static final long MAX_WAIT = 60 * 1000;<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span>    final FileStatus target;<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    volatile boolean result;<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    volatile boolean setFromCleaner = false;<a name="line.186"></a>
+<span class="sourceLineNo">180</span>  @Override<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  public synchronized void cancel(boolean mayInterruptIfRunning) {<a name="line.181"></a>
+<span class="sourceLineNo">182</span>    super.cancel(mayInterruptIfRunning);<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    for (Thread t : oldWALsCleaner) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      t.interrupt();<a name="line.184"></a>
+<span class="sourceLineNo">185</span>    }<a name="line.185"></a>
+<span class="sourceLineNo">186</span>  }<a name="line.186"></a>
 <span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    static CleanerContext createCleanerContext(FileStatus status) {<a name="line.188"></a>
-<span class="sourceLineNo">189</span>      return status != null ? new CleanerContext(status) : null;<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    }<a name="line.190"></a>
+<span class="sourceLineNo">188</span>  private static final class CleanerContext {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>    // At most waits 60 seconds<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    static final long MAX_WAIT = 60 * 1000;<a name="line.190"></a>
 <span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span>    private CleanerContext(FileStatus status) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      this.target = status;<a name="line.193"></a>
-<span class="sourceLineNo">194</span>      this.result = false;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    }<a name="line.195"></a>
-<span class="sourceLineNo">196</span><a name="line.196"></a>
-<span class="sourceLineNo">197</span>    synchronized void setResult(boolean res) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      this.result = res;<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      this.setFromCleaner = true;<a name="line.199"></a>
-<span class="sourceLineNo">200</span>      notify();<a name="line.200"></a>
-<span class="sourceLineNo">201</span>    }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>    synchronized boolean getResult(long waitIfNotFinished) {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>      long totalTime = 0;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      try {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>        while (!setFromCleaner) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span>          wait(waitIfNotFinished);<a name="line.207"></a>
-<span class="sourceLineNo">208</span>          totalTime += waitIfNotFinished;<a name="line.208"></a>
-<span class="sourceLineNo">209</span>          if (totalTime &gt;= MAX_WAIT) {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>            LOG.warn("Spend too much time to delete oldwals " + target);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>            return result;<a name="line.211"></a>
-<span class="sourceLineNo">212</span>          }<a name="line.212"></a>
-<span class="sourceLineNo">213</span>        }<a name="line.213"></a>
-<span class="sourceLineNo">214</span>      } catch (InterruptedException e) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>        LOG.warn("Interrupted while waiting deletion of " + target);<a name="line.215"></a>
-<span class="sourceLineNo">216</span>        return result;<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      }<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      return result;<a name="line.218"></a>
-<span class="sourceLineNo">219</span>    }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>    FileStatus getTargetToClean() {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>      return target;<a name="line.222"></a>
-<span class="sourceLineNo">223</span>    }<a name="line.223"></a>
-<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>}<a name="line.225"></a>
+<span class="sourceLineNo">192</span>    final FileStatus target;<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    volatile boolean result;<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    volatile boolean setFromCleaner = false;<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span>    static CleanerContext createCleanerContext(FileStatus status) {<a name="line.196"></a>
+<span class="sourceLineNo">197</span>      return status != null ? new CleanerContext(status) : null;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    }<a name="line.198"></a>
+<span class="sourceLineNo">199</span><a name="line.199"></a>
+<span class="sourceLineNo">200</span>    private CleanerContext(FileStatus status) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>      this.target = status;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>      this.result = false;<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    }<a name="line.203"></a>
+<span class="sourceLineNo">204</span><a name="line.204"></a>
+<span class="sourceLineNo">205</span>    synchronized void setResult(boolean res) {<a name="line.205"></a>
+<span class="sourceLineNo">206</span>      this.result = res;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>      this.setFromCleaner = true;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      notify();<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span>    synchronized boolean getResult(long waitIfNotFinished) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>      long totalTime = 0;<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      try {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>        while (!setFromCleaner) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>          wait(waitIfNotFinished);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>          totalTime += waitIfNotFinished;<a name="line.216"></a>
+<span class="sourceLineNo">217</span>          if (totalTime &gt;= MAX_WAIT) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>            LOG.warn("Spend too much time to delete oldwals " + target);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>            return result;<a name="line.219"></a>
+<span class="sourceLineNo">220</span>          }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>        }<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      } catch (InterruptedException e) {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>        LOG.warn("Interrupted while waiting deletion of " + target);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>        return result;<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      }<a name="line.225"></a>
+<span class="sourceLineNo">226</span>      return result;<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    }<a name="line.227"></a>
+<span class="sourceLineNo">228</span><a name="line.228"></a>
+<span class="sourceLineNo">229</span>    FileStatus getTargetToClean() {<a name="line.229"></a>
+<span class="sourceLineNo">230</span>      return target;<a name="line.230"></a>
+<span class="sourceLineNo">231</span>    }<a name="line.231"></a>
+<span class="sourceLineNo">232</span>  }<a name="line.232"></a>
+<span class="sourceLineNo">233</span>}<a name="line.233"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.html
index f02bbe3..6d501ca 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/cleaner/LogCleaner.html
@@ -185,52 +185,60 @@
 <span class="sourceLineNo">177</span>    }<a name="line.177"></a>
 <span class="sourceLineNo">178</span>  }<a name="line.178"></a>
 <span class="sourceLineNo">179</span><a name="line.179"></a>
-<span class="sourceLineNo">180</span>  private static final class CleanerContext {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    // At most waits 60 seconds<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    static final long MAX_WAIT = 60 * 1000;<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span>    final FileStatus target;<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    volatile boolean result;<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    volatile boolean setFromCleaner = false;<a name="line.186"></a>
+<span class="sourceLineNo">180</span>  @Override<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  public synchronized void cancel(boolean mayInterruptIfRunning) {<a name="line.181"></a>
+<span class="sourceLineNo">182</span>    super.cancel(mayInterruptIfRunning);<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    for (Thread t : oldWALsCleaner) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      t.interrupt();<a name="line.184"></a>
+<span class="sourceLineNo">185</span>    }<a name="line.185"></a>
+<span class="sourceLineNo">186</span>  }<a name="line.186"></a>
 <span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    static CleanerContext createCleanerContext(FileStatus status) {<a name="line.188"></a>
-<span class="sourceLineNo">189</span>      return status != null ? new CleanerContext(status) : null;<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    }<a name="line.190"></a>
+<span class="sourceLineNo">188</span>  private static final class CleanerContext {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>    // At most waits 60 seconds<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    static final long MAX_WAIT = 60 * 1000;<a name="line.190"></a>
 <span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span>    private CleanerContext(FileStatus status) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      this.target = status;<a name="line.193"></a>
-<span class="sourceLineNo">194</span>      this.result = false;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    }<a name="line.195"></a>
-<span class="sourceLineNo">196</span><a name="line.196"></a>
-<span class="sourceLineNo">197</span>    synchronized void setResult(boolean res) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      this.result = res;<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      this.setFromCleaner = true;<a name="line.199"></a>
-<span class="sourceLineNo">200</span>      notify();<a name="line.200"></a>
-<span class="sourceLineNo">201</span>    }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>    synchronized boolean getResult(long waitIfNotFinished) {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>      long totalTime = 0;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      try {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>        while (!setFromCleaner) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span>          wait(waitIfNotFinished);<a name="line.207"></a>
-<span class="sourceLineNo">208</span>          totalTime += waitIfNotFinished;<a name="line.208"></a>
-<span class="sourceLineNo">209</span>          if (totalTime &gt;= MAX_WAIT) {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>            LOG.warn("Spend too much time to delete oldwals " + target);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>            return result;<a name="line.211"></a>
-<span class="sourceLineNo">212</span>          }<a name="line.212"></a>
-<span class="sourceLineNo">213</span>        }<a name="line.213"></a>
-<span class="sourceLineNo">214</span>      } catch (InterruptedException e) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>        LOG.warn("Interrupted while waiting deletion of " + target);<a name="line.215"></a>
-<span class="sourceLineNo">216</span>        return result;<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      }<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      return result;<a name="line.218"></a>
-<span class="sourceLineNo">219</span>    }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>    FileStatus getTargetToClean() {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>      return target;<a name="line.222"></a>
-<span class="sourceLineNo">223</span>    }<a name="line.223"></a>
-<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>}<a name="line.225"></a>
+<span class="sourceLineNo">192</span>    final FileStatus target;<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    volatile boolean result;<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    volatile boolean setFromCleaner = false;<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span>    static CleanerContext createCleanerContext(FileStatus status) {<a name="line.196"></a>
+<span class="sourceLineNo">197</span>      return status != null ? new CleanerContext(status) : null;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    }<a name="line.198"></a>
+<span class="sourceLineNo">199</span><a name="line.199"></a>
+<span class="sourceLineNo">200</span>    private CleanerContext(FileStatus status) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>      this.target = status;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>      this.result = false;<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    }<a name="line.203"></a>
+<span class="sourceLineNo">204</span><a name="line.204"></a>
+<span class="sourceLineNo">205</span>    synchronized void setResult(boolean res) {<a name="line.205"></a>
+<span class="sourceLineNo">206</span>      this.result = res;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>      this.setFromCleaner = true;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      notify();<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span>    synchronized boolean getResult(long waitIfNotFinished) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>      long totalTime = 0;<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      try {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>        while (!setFromCleaner) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>          wait(waitIfNotFinished);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>          totalTime += waitIfNotFinished;<a name="line.216"></a>
+<span class="sourceLineNo">217</span>          if (totalTime &gt;= MAX_WAIT) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>            LOG.warn("Spend too much time to delete oldwals " + target);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>            return result;<a name="line.219"></a>
+<span class="sourceLineNo">220</span>          }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>        }<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      } catch (InterruptedException e) {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>        LOG.warn("Interrupted while waiting deletion of " + target);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>        return result;<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      }<a name="line.225"></a>
+<span class="sourceLineNo">226</span>      return result;<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    }<a name="line.227"></a>
+<span class="sourceLineNo">228</span><a name="line.228"></a>
+<span class="sourceLineNo">229</span>    FileStatus getTargetToClean() {<a name="line.229"></a>
+<span class="sourceLineNo">230</span>      return target;<a name="line.230"></a>
+<span class="sourceLineNo">231</span>    }<a name="line.231"></a>
+<span class="sourceLineNo">232</span>  }<a name="line.232"></a>
+<span class="sourceLineNo">233</span>}<a name="line.233"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.html
index 4b02458..7bad604 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.html
@@ -203,9 +203,9 @@
 <span class="sourceLineNo">195</span>    // Assign meta if still carrying it. Check again: region may be assigned because of RIT timeout<a name="line.195"></a>
 <span class="sourceLineNo">196</span>    final AssignmentManager am = env.getMasterServices().getAssignmentManager();<a name="line.196"></a>
 <span class="sourceLineNo">197</span>    for (RegionInfo hri: am.getRegionStates().getServerRegionInfoSet(serverName)) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      if (!isDefaultMetaRegion(hri)) continue;<a name="line.198"></a>
-<span class="sourceLineNo">199</span><a name="line.199"></a>
-<span class="sourceLineNo">200</span>      am.offlineRegion(hri);<a name="line.200"></a>
+<span class="sourceLineNo">198</span>      if (!isDefaultMetaRegion(hri)) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>        continue;<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      }<a name="line.200"></a>
 <span class="sourceLineNo">201</span>      addChildProcedure(new RecoverMetaProcedure(serverName, this.shouldSplitWal));<a name="line.201"></a>
 <span class="sourceLineNo">202</span>    }<a name="line.202"></a>
 <span class="sourceLineNo">203</span>  }<a name="line.203"></a>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
index 7271567..a08bfa4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
@@ -518,10 +518,10 @@
 <span class="sourceLineNo">510</span>    // We have numThreads executor + one timer thread used for timing out<a name="line.510"></a>
 <span class="sourceLineNo">511</span>    // procedures and triggering periodic procedures.<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    this.corePoolSize = numThreads;<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    LOG.info("Starting ProcedureExecutor Worker threads (ProcExecWrkr)=" + corePoolSize);<a name="line.513"></a>
+<span class="sourceLineNo">513</span>    LOG.info("Starting ProcedureExecutor Worker threads (ProcedureExecutorWorker)=" + corePoolSize);<a name="line.513"></a>
 <span class="sourceLineNo">514</span><a name="line.514"></a>
 <span class="sourceLineNo">515</span>    // Create the Thread Group for the executors<a name="line.515"></a>
-<span class="sourceLineNo">516</span>    threadGroup = new ThreadGroup("ProcExecThrdGrp");<a name="line.516"></a>
+<span class="sourceLineNo">516</span>    threadGroup = new ThreadGroup("ProcedureExecutorWorkerGroup");<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
 <span class="sourceLineNo">518</span>    // Create the timeout executor<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    timeoutExecutor = new TimeoutExecutorThread(threadGroup);<a name="line.519"></a>
@@ -600,7 +600,7 @@
 <span class="sourceLineNo">592</span>    try {<a name="line.592"></a>
 <span class="sourceLineNo">593</span>      threadGroup.destroy();<a name="line.593"></a>
 <span class="sourceLineNo">594</span>    } catch (IllegalThreadStateException e) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.error("Thread group " + threadGroup + " contains running threads");<a name="line.595"></a>
+<span class="sourceLineNo">595</span>      LOG.error("ThreadGroup " + threadGroup + " contains running threads; " + e.getMessage());<a name="line.595"></a>
 <span class="sourceLineNo">596</span>      threadGroup.list();<a name="line.596"></a>
 <span class="sourceLineNo">597</span>    } finally {<a name="line.597"></a>
 <span class="sourceLineNo">598</span>      threadGroup = null;<a name="line.598"></a>
@@ -1717,7 +1717,7 @@
 <span class="sourceLineNo">1709</span>    private Procedure activeProcedure;<a name="line.1709"></a>
 <span class="sourceLineNo">1710</span><a name="line.1710"></a>
 <span class="sourceLineNo">1711</span>    public WorkerThread(final ThreadGroup group) {<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span>      super(group, "ProcExecWrkr-" + workerId.incrementAndGet());<a name="line.1712"></a>
+<span class="sourceLineNo">1712</span>      super(group, "ProcedureExecutorWorker-" + workerId.incrementAndGet());<a name="line.1712"></a>
 <span class="sourceLineNo">1713</span>      setDaemon(true);<a name="line.1713"></a>
 <span class="sourceLineNo">1714</span>    }<a name="line.1714"></a>
 <span class="sourceLineNo">1715</span><a name="line.1715"></a>
@@ -1760,7 +1760,7 @@
 <span class="sourceLineNo">1752</span>      } catch (Throwable t) {<a name="line.1752"></a>
 <span class="sourceLineNo">1753</span>        LOG.warn("Worker terminating UNNATURALLY " + this.activeProcedure, t);<a name="line.1753"></a>
 <span class="sourceLineNo">1754</span>      } finally {<a name="line.1754"></a>
-<span class="sourceLineNo">1755</span>        LOG.debug("Worker terminated.");<a name="line.1755"></a>
+<span class="sourceLineNo">1755</span>        LOG.trace("Worker terminated.");<a name="line.1755"></a>
 <span class="sourceLineNo">1756</span>      }<a name="line.1756"></a>
 <span class="sourceLineNo">1757</span>      workerThreads.remove(this);<a name="line.1757"></a>
 <span class="sourceLineNo">1758</span>    }<a name="line.1758"></a>
@@ -1912,120 +1912,123 @@
 <span class="sourceLineNo">1904</span>        for (int i = 0; isAlive(); ++i) {<a name="line.1904"></a>
 <span class="sourceLineNo">1905</span>          sendStopSignal();<a name="line.1905"></a>
 <span class="sourceLineNo">1906</span>          join(250);<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>          if (i &gt; 0 &amp;&amp; (i % 8) == 0) {<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>            LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>              StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>          }<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span>        }<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>      } catch (InterruptedException e) {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span>    }<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>  }<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span><a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>  // ==========================================================================<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>  //  Inline Chores (executors internal chores)<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>  // ==========================================================================<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span>    private long timeout;<a name="line.1922"></a>
-<span class="sourceLineNo">1923</span><a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>    public abstract int getTimeoutInterval();<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span><a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>    protected void refreshTimeout() {<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>      this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    }<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span><a name="line.1929"></a>
-<span class="sourceLineNo">1930</span>    @Override<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    public long getTimeout() {<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>      return timeout;<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>    }<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>  }<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span><a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>  // ----------------------------------------------------------------------------<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span>  // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>  // full set of procedures pending and completed to write a compacted<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>  // version of the log (in case is a log)?<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>  // In theory no, procedures are have a short life, so at some point the store<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>  // will have the tracker saying everything is in the last log.<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>  // ----------------------------------------------------------------------------<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span><a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>  private final class WorkerMonitor extends InlineChore {<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>    public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>        "hbase.procedure.worker.monitor.interval.msec";<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>    public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>        "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>    private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>    public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        "hbase.procedure.worker.add.stuck.percentage";<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>    private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span><a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>    private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>    private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span><a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>    public WorkerMonitor() {<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>      refreshConfig();<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>    }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span><a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>    @Override<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>    public void run() {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>      final int stuckCount = checkForStuckWorkers();<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>      checkThreadCount(stuckCount);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span><a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>      // refresh interval (poor man dynamic conf update)<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>      refreshConfig();<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>    }<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span><a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    private int checkForStuckWorkers() {<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span>      // check if any of the worker is stuck<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>      int stuckCount = 0;<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      for (WorkerThread worker: workerThreads) {<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        if (worker.getCurrentRunTime() &lt; stuckThreshold) {<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>          continue;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>        }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>        // WARN the worker is stuck<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>        stuckCount++;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>        LOG.warn("Worker stuck " + worker +<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span>            " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>      }<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      return stuckCount;<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>    }<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span><a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>    private void checkThreadCount(final int stuckCount) {<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // nothing to do if there are no runnable tasks<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      if (stuckCount &lt; 1 || !scheduler.hasRunnables()) return;<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span><a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span>      // and every handler is active.<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      if (stuckPerc &gt;= addWorkerStuckPercentage &amp;&amp;<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>          activeExecutorCount.get() == workerThreads.size()) {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        workerThreads.add(worker);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        worker.start();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>        LOG.debug("Added new worker thread " + worker);<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>      }<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>    }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span><a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>    private void refreshConfig() {<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>          DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>        DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span><a name="line.2014"></a>
-<span class="sourceLineNo">2015</span>    @Override<a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>    public int getTimeoutInterval() {<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>      return timeoutInterval;<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>    }<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>  }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>}<a name="line.2020"></a>
+<span class="sourceLineNo">1907</span>          // Log every two seconds; send interrupt too.<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>          if (i &gt; 0 &amp;&amp; (i % 8) == 0) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>            LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>              StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime) +<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>            "; sending interrupt");<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>            interrupt();<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span>          }<a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        }<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>      } catch (InterruptedException e) {<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>        LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span>      }<a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>    }<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>  }<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span><a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>  // ==========================================================================<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>  //  Inline Chores (executors internal chores)<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  // ==========================================================================<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span>  private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>    private long timeout;<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span><a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>    public abstract int getTimeoutInterval();<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span><a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>    protected void refreshTimeout() {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>      this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>    }<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    @Override<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    public long getTimeout() {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      return timeout;<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>  }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span>  // ----------------------------------------------------------------------------<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>  // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>  // full set of procedures pending and completed to write a compacted<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>  // version of the log (in case is a log)?<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>  // In theory no, procedures are have a short life, so at some point the store<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>  // will have the tracker saying everything is in the last log.<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>  // ----------------------------------------------------------------------------<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span><a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>  private final class WorkerMonitor extends InlineChore {<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>        "hbase.procedure.worker.monitor.interval.msec";<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>    private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span><a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>    public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>        "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>    private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>    public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        "hbase.procedure.worker.add.stuck.percentage";<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>    private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>    private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>    private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>    private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span><a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>    public WorkerMonitor() {<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      refreshConfig();<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>    }<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span><a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>    @Override<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>    public void run() {<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>      final int stuckCount = checkForStuckWorkers();<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>      checkThreadCount(stuckCount);<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span><a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>      // refresh interval (poor man dynamic conf update)<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>      refreshConfig();<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>    }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span><a name="line.1976"></a>
+<span class="sourceLineNo">1977</span>    private int checkForStuckWorkers() {<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>      // check if any of the worker is stuck<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      int stuckCount = 0;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>      for (WorkerThread worker: workerThreads) {<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>        if (worker.getCurrentRunTime() &lt; stuckThreshold) {<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>          continue;<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span>        }<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span><a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>        // WARN the worker is stuck<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>        stuckCount++;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span>        LOG.warn("Worker stuck " + worker +<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>            " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      }<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      return stuckCount;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>    }<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span><a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>    private void checkThreadCount(final int stuckCount) {<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // nothing to do if there are no runnable tasks<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      if (stuckCount &lt; 1 || !scheduler.hasRunnables()) return;<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span><a name="line.1996"></a>
+<span class="sourceLineNo">1997</span>      // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      // and every handler is active.<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      if (stuckPerc &gt;= addWorkerStuckPercentage &amp;&amp;<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>          activeExecutorCount.get() == workerThreads.size()) {<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        workerThreads.add(worker);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>        worker.start();<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        LOG.debug("Added new worker thread " + worker);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>    }<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span><a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>    private void refreshConfig() {<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>      addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>          DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>      timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>        DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>        DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>    }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>    @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>    public int getTimeoutInterval() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      return timeoutInterval;<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>    }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>  }<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>}<a name="line.2023"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
index 7271567..a08bfa4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
@@ -518,10 +518,10 @@
 <span class="sourceLineNo">510</span>    // We have numThreads executor + one timer thread used for timing out<a name="line.510"></a>
 <span class="sourceLineNo">511</span>    // procedures and triggering periodic procedures.<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    this.corePoolSize = numThreads;<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    LOG.info("Starting ProcedureExecutor Worker threads (ProcExecWrkr)=" + corePoolSize);<a name="line.513"></a>
+<span class="sourceLineNo">513</span>    LOG.info("Starting ProcedureExecutor Worker threads (ProcedureExecutorWorker)=" + corePoolSize);<a name="line.513"></a>
 <span class="sourceLineNo">514</span><a name="line.514"></a>
 <span class="sourceLineNo">515</span>    // Create the Thread Group for the executors<a name="line.515"></a>
-<span class="sourceLineNo">516</span>    threadGroup = new ThreadGroup("ProcExecThrdGrp");<a name="line.516"></a>
+<span class="sourceLineNo">516</span>    threadGroup = new ThreadGroup("ProcedureExecutorWorkerGroup");<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
 <span class="sourceLineNo">518</span>    // Create the timeout executor<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    timeoutExecutor = new TimeoutExecutorThread(threadGroup);<a name="line.519"></a>
@@ -600,7 +600,7 @@
 <span class="sourceLineNo">592</span>    try {<a name="line.592"></a>
 <span class="sourceLineNo">593</span>      threadGroup.destroy();<a name="line.593"></a>
 <span class="sourceLineNo">594</span>    } catch (IllegalThreadStateException e) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.error("Thread group " + threadGroup + " contains running threads");<a name="line.595"></a>
+<span class="sourceLineNo">595</span>      LOG.error("ThreadGroup " + threadGroup + " contains running threads; " + e.getMessage());<a name="line.595"></a>
 <span class="sourceLineNo">596</span>      threadGroup.list();<a name="line.596"></a>
 <span class="sourceLineNo">597</span>    } finally {<a name="line.597"></a>
 <span class="sourceLineNo">598</span>      threadGroup = null;<a name="line.598"></a>
@@ -1717,7 +1717,7 @@
 <span class="sourceLineNo">1709</span>    private Procedure activeProcedure;<a name="line.1709"></a>
 <span class="sourceLineNo">1710</span><a name="line.1710"></a>
 <span class="sourceLineNo">1711</span>    public WorkerThread(final ThreadGroup group) {<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span>      super(group, "ProcExecWrkr-" + workerId.incrementAndGet());<a name="line.1712"></a>
+<span class="sourceLineNo">1712</span>      super(group, "ProcedureExecutorWorker-" + workerId.incrementAndGet());<a name="line.1712"></a>
 <span class="sourceLineNo">1713</span>      setDaemon(true);<a name="line.1713"></a>
 <span class="sourceLineNo">1714</span>    }<a name="line.1714"></a>
 <span class="sourceLineNo">1715</span><a name="line.1715"></a>
@@ -1760,7 +1760,7 @@
 <span class="sourceLineNo">1752</span>      } catch (Throwable t) {<a name="line.1752"></a>
 <span class="sourceLineNo">1753</span>        LOG.warn("Worker terminating UNNATURALLY " + this.activeProcedure, t);<a name="line.1753"></a>
 <span class="sourceLineNo">1754</span>      } finally {<a name="line.1754"></a>
-<span class="sourceLineNo">1755</span>        LOG.debug("Worker terminated.");<a name="line.1755"></a>
+<span class="sourceLineNo">1755</span>        LOG.trace("Worker terminated.");<a name="line.1755"></a>
 <span class="sourceLineNo">1756</span>      }<a name="line.1756"></a>
 <span class="sourceLineNo">1757</span>      workerThreads.remove(this);<a name="line.1757"></a>
 <span class="sourceLineNo">1758</span>    }<a name="line.1758"></a>
@@ -1912,120 +1912,123 @@
 <span class="sourceLineNo">1904</span>        for (int i = 0; isAlive(); ++i) {<a name="line.1904"></a>
 <span class="sourceLineNo">1905</span>          sendStopSignal();<a name="line.1905"></a>
 <span class="sourceLineNo">1906</span>          join(250);<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>          if (i &gt; 0 &amp;&amp; (i % 8) == 0) {<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>            LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>              StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>          }<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span>        }<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>      } catch (InterruptedException e) {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span>    }<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>  }<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span><a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>  // ==========================================================================<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>  //  Inline Chores (executors internal chores)<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>  // ==========================================================================<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span>    private long timeout;<a name="line.1922"></a>
-<span class="sourceLineNo">1923</span><a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>    public abstract int getTimeoutInterval();<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span><a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>    protected void refreshTimeout() {<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>      this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    }<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span><a name="line.1929"></a>
-<span class="sourceLineNo">1930</span>    @Override<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    public long getTimeout() {<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>      return timeout;<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>    }<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>  }<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span><a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>  // ----------------------------------------------------------------------------<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span>  // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>  // full set of procedures pending and completed to write a compacted<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>  // version of the log (in case is a log)?<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>  // In theory no, procedures are have a short life, so at some point the store<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>  // will have the tracker saying everything is in the last log.<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>  // ----------------------------------------------------------------------------<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span><a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>  private final class WorkerMonitor extends InlineChore {<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>    public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>        "hbase.procedure.worker.monitor.interval.msec";<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>    public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>        "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>    private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>    public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        "hbase.procedure.worker.add.stuck.percentage";<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>    private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span><a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>    private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>    private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span><a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>    public WorkerMonitor() {<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>      refreshConfig();<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>    }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span><a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>    @Override<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>    public void run() {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>      final int stuckCount = checkForStuckWorkers();<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>      checkThreadCount(stuckCount);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span><a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>      // refresh interval (poor man dynamic conf update)<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>      refreshConfig();<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>    }<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span><a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    private int checkForStuckWorkers() {<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span>      // check if any of the worker is stuck<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>      int stuckCount = 0;<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      for (WorkerThread worker: workerThreads) {<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        if (worker.getCurrentRunTime() &lt; stuckThreshold) {<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>          continue;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>        }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>        // WARN the worker is stuck<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>        stuckCount++;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>        LOG.warn("Worker stuck " + worker +<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span>            " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>      }<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      return stuckCount;<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>    }<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span><a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>    private void checkThreadCount(final int stuckCount) {<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // nothing to do if there are no runnable tasks<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      if (stuckCount &lt; 1 || !scheduler.hasRunnables()) return;<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span><a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span>      // and every handler is active.<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      if (stuckPerc &gt;= addWorkerStuckPercentage &amp;&amp;<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>          activeExecutorCount.get() == workerThreads.size()) {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        workerThreads.add(worker);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        worker.start();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>        LOG.debug("Added new worker thread " + worker);<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>      }<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>    }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span><a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>    private void refreshConfig() {<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>          DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>        DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span><a name="line.2014"></a>
-<span class="sourceLineNo">2015</span>    @Override<a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>    public int getTimeoutInterval() {<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>      return timeoutInterval;<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>    }<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>  }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>}<a name="line.2020"></a>
+<span class="sourceLineNo">1907</span>          // Log every two seconds; send interrupt too.<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>          if (i &gt; 0 &amp;&amp; (i % 8) == 0) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>            LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>              StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime) +<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>            "; sending interrupt");<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>            interrupt();<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span>          }<a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        }<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>      } catch (InterruptedException e) {<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>        LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span>      }<a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>    }<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>  }<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span><a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>  // ==========================================================================<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>  //  Inline Chores (executors internal chores)<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  // ==========================================================================<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span>  private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>    private long timeout;<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span><a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>    public abstract int getTimeoutInterval();<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span><a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>    protected void refreshTimeout() {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>      this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>    }<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    @Override<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    public long getTimeout() {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      return timeout;<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>  }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span>  // ----------------------------------------------------------------------------<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>  // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>  // full set of procedures pending and completed to write a compacted<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>  // version of the log (in case is a log)?<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>  // In theory no, procedures are have a short life, so at some point the store<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>  // will have the tracker saying everything is in the last log.<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>  // ----------------------------------------------------------------------------<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span><a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>  private final class WorkerMonitor extends InlineChore {<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>        "hbase.procedure.worker.monitor.interval.msec";<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>    private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span><a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>    public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>        "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>    private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>    public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        "hbase.procedure.worker.add.stuck.percentage";<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>    private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>    private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>    private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>    private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span><a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>    public WorkerMonitor() {<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      refreshConfig();<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>    }<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span><a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>    @Override<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>    public void run() {<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>      final int stuckCount = checkForStuckWorkers();<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>      checkThreadCount(stuckCount);<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span><a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>      // refresh interval (poor man dynamic conf update)<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>      refreshConfig();<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>    }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span><a name="line.1976"></a>
+<span class="sourceLineNo">1977</span>    private int checkForStuckWorkers() {<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>      // check if any of the worker is stuck<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      int stuckCount = 0;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>      for (WorkerThread worker: workerThreads) {<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>        if (worker.getCurrentRunTime() &lt; stuckThreshold) {<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>          continue;<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span>        }<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span><a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>        // WARN the worker is stuck<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>        stuckCount++;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span>        LOG.warn("Worker stuck " + worker +<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>            " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      }<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      return stuckCount;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>    }<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span><a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>    private void checkThreadCount(final int stuckCount) {<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // nothing to do if there are no runnable tasks<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      if (stuckCount &lt; 1 || !scheduler.hasRunnables()) return;<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span><a name="line.1996"></a>
+<span class="sourceLineNo">1997</span>      // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      // and every handler is active.<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      if (stuckPerc &gt;= addWorkerStuckPercentage &amp;&amp;<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>          activeExecutorCount.get() == workerThreads.size()) {<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        workerThreads.add(worker);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>        worker.start();<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        LOG.debug("Added new worker thread " + worker);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>    }<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span><a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>    private void refreshConfig() {<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>      addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>          DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>      timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>        DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>        DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>    }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>    @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>    public int getTimeoutInterval() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      return timeoutInterval;<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>    }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>  }<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>}<a name="line.2023"></a>
 
 
 


Mime
View raw message