asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Michael Blow (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Timeout NC thread dump requests after 60 seconds
Date Mon, 26 Sep 2016 04:48:17 GMT
Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1213

Change subject: Timeout NC thread dump requests after 60 seconds
......................................................................

Timeout NC thread dump requests after 60 seconds

Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a
---
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
3 files changed, 36 insertions(+), 2 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/13/1213/1

diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index be53232..7931cf8 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -20,6 +20,10 @@
 
 import java.lang.management.ManagementFactory;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import org.apache.hyracks.control.cc.ClusterControllerService;
 import org.apache.hyracks.control.cc.NodeControllerState;
@@ -27,6 +31,9 @@
 import org.apache.hyracks.control.common.work.ThreadDumpWork;
 
 public class GetThreadDumpWork extends ThreadDumpWork {
+    private static final Logger LOGGER = Logger.getLogger(ThreadDumpWork.class.getName());
+    public static final int TIMEOUT_SECS = 60;
+
     private final ClusterControllerService ccs;
     private final String nodeId;
     private final IResultCallback<String> callback;
@@ -55,8 +62,27 @@
                 try {
                     ncState.getNodeController().takeThreadDump(run.getRequestId());
                 } catch (Exception e) {
+                    ccs.removeThreadDumpRun(run.getRequestId());
                     callback.setException(e);
                 }
+                final long requestTime = System.currentTimeMillis();
+                ccs.getExecutor().execute(() -> {
+                    try {
+                        final long queueTime = System.currentTimeMillis() - requestTime;
+                        final long sleepTime = TimeUnit.SECONDS.toMillis(TIMEOUT_SECS) -
queueTime;
+                        if (sleepTime > 0) {
+                            Thread.sleep(sleepTime);
+                        }
+                        if (ccs.removeThreadDumpRun(run.getRequestId()) != null) {
+                            LOGGER.log(Level.WARNING, "Timed out thread dump request " +
run.getRequestId()
+                                    + " for node " + nodeId);
+                            callback.setException(new TimeoutException("Thread dump request
for node " + nodeId
+                                    + " timed out after " + TIMEOUT_SECS + " seconds."));
+                        }
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                });
             }
         }
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
index bbdf211..2dae4b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
@@ -18,10 +18,13 @@
  */
 package org.apache.hyracks.control.cc.work;
 
+import java.util.logging.Logger;
+
 import org.apache.hyracks.control.cc.ClusterControllerService;
 import org.apache.hyracks.control.common.work.AbstractWork;
 
 public class NotifyThreadDumpResponse extends AbstractWork {
+    private static final Logger LOGGER = Logger.getLogger(NotifyThreadDumpResponse.class.getName());
 
     private final ClusterControllerService ccs;
 
@@ -36,6 +39,12 @@
 
     @Override
     public void run() {
-        ccs.removeThreadDumpRun(requestId).notifyThreadDumpReceived(threadDumpJSON);
+        LOGGER.fine("Delivering thread dump response: " + requestId);
+        final GetThreadDumpWork.ThreadDumpRun threadDumpRun = ccs.removeThreadDumpRun(requestId);
+        if (threadDumpRun == null) {
+            LOGGER.warning("Thread dump run " + requestId + " not found; discarding reply:
" + threadDumpJSON);
+        } else {
+            threadDumpRun.notifyThreadDumpReceived(threadDumpJSON);
+        }
     }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
index 1fc4690..85233b2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
@@ -33,7 +33,6 @@
     @Override
     protected void doRun() throws Exception {
         final String result = takeDump(ncs.getThreadMXBean());
-
         ncs.getClusterController().notifyThreadDump(
                 ncs.getApplicationContext().getNodeId(), requestId, result);
     }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1213
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mblow@apache.org>

Mime
View raw message