asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject asterixdb git commit: [NO ISSUE][HYR][*DB] Exit JVM from independent thread to avoid deadlocks
Date Sat, 26 Aug 2017 17:54:02 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master b3637a6a2 -> d2f4fbf9f


[NO ISSUE][HYR][*DB] Exit JVM from independent thread to avoid deadlocks

Change-Id: I21b2090ea3ef85e95ae90de04b08b4a6d22ebe42
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1973
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d2f4fbf9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d2f4fbf9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d2f4fbf9

Branch: refs/heads/master
Commit: d2f4fbf9feed9fa75e7f4e27224bad11905ced33
Parents: b3637a6
Author: Michael Blow <mblow@apache.org>
Authored: Sat Aug 26 00:09:22 2017 -0400
Committer: Michael Blow <mblow@apache.org>
Committed: Sat Aug 26 10:53:39 2017 -0700

----------------------------------------------------------------------
 .../message/StartupTaskResponseMessage.java     |  3 +-
 .../lifecycle/LifeCycleComponentManager.java    |  4 +-
 .../hyracks-control/hyracks-control-cc/pom.xml  |  5 ++
 .../control/cc/ClusterControllerService.java    |  5 ++
 .../control/cc/work/ClusterShutdownWork.java    | 42 ++++++-------
 .../hyracks-control/hyracks-control-nc/pom.xml  |  5 ++
 .../control/nc/NodeControllerService.java       |  5 ++
 .../hyracks/control/nc/task/ShutdownTask.java   | 11 +---
 .../java/org/apache/hyracks/util/ExitUtil.java  | 63 ++++++++++++++++++++
 9 files changed, 109 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java
index 9125857..aaf8918 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/StartupTaskResponseMessage.java
@@ -30,6 +30,7 @@ import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.service.IControllerService;
 import org.apache.hyracks.control.nc.NCShutdownHook;
+import org.apache.hyracks.util.ExitUtil;
 
 public class StartupTaskResponseMessage implements INCLifecycleMessage, INcAddressedMessage
{
 
@@ -69,7 +70,7 @@ public class StartupTaskResponseMessage implements INCLifecycleMessage,
INcAddre
         } finally {
             if (!success) {
                 // stop NC so that it can be started again
-                Runtime.getRuntime().exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE); //NOSONAR
startup failed
+                ExitUtil.exit(NCShutdownHook.FAILED_TO_STARTUP_EXIT_CODE);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
index 76fa322..f5b4417 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/lifecycle/LifeCycleComponentManager.java
@@ -28,6 +28,8 @@ import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.hyracks.util.ExitUtil;
+
 public class LifeCycleComponentManager implements ILifeCycleComponentManager {
 
     public static final class Config {
@@ -54,7 +56,7 @@ public class LifeCycleComponentManager implements ILifeCycleComponentManager
{
         try {
             LOGGER.log(Level.SEVERE, "Uncaught Exception from thread " + t.getName() + ".
Calling shutdown hook", e);
         } finally {
-            Runtime.getRuntime().exit(99);// NOSONAR: It is really required
+            ExitUtil.exit(99);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
index b28cc79..fce37dd 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
@@ -43,6 +43,11 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-util</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-all</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
index d32e577..a243bf8 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
@@ -83,6 +83,7 @@ import org.apache.hyracks.control.common.work.WorkQueue;
 import org.apache.hyracks.ipc.api.IIPCI;
 import org.apache.hyracks.ipc.impl.IPCSystem;
 import org.apache.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+import org.apache.hyracks.util.ExitUtil;
 import org.xml.sax.InputSource;
 
 public class ClusterControllerService implements IControllerService {
@@ -138,6 +139,10 @@ public class ClusterControllerService implements IControllerService {
 
     private ShutdownRun shutdownCallback;
 
+    static {
+        ExitUtil.init();
+    }
+
     public ClusterControllerService(final CCConfig config) throws Exception {
         this(config, getApplication(config));
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
index 0b89f55..613efad 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java
@@ -30,6 +30,7 @@ import org.apache.hyracks.control.common.shutdown.ShutdownRun;
 import org.apache.hyracks.control.common.work.IResultCallback;
 import org.apache.hyracks.control.common.work.SynchronizableWork;
 import org.apache.hyracks.ipc.exceptions.IPCException;
+import org.apache.hyracks.util.ExitUtil;
 
 public class ClusterShutdownWork extends SynchronizableWork {
     private static final Logger LOGGER = Logger.getLogger(ClusterShutdownWork.class.getName());
@@ -53,41 +54,36 @@ public class ClusterShutdownWork extends SynchronizableWork {
             }
             INodeManager nodeManager = ccs.getNodeManager();
             Collection<String> nodeIds = nodeManager.getAllNodeIds();
-            /**
+            /*
              * set up our listener for the node ACKs
              */
             final ShutdownRun shutdownStatus = new ShutdownRun(nodeIds);
             // set up the CC to listen for it
             ccs.setShutdownRun(shutdownStatus);
-            /**
+            /*
              * Shutdown all the nodes...
              */
             nodeManager.apply(this::shutdownNode);
 
-            ccs.getExecutor().execute(new Runnable() {
-                @Override
-                public void run() {
-                    try {
+            ccs.getExecutor().execute(() -> {
+                try {
+                    /*
+                     * wait for all our acks
+                     */
+                    LOGGER.info("Waiting for NCs to shutdown...");
+                    boolean cleanShutdown = shutdownStatus.waitForCompletion();
+                    if (!cleanShutdown) {
                         /*
-                         * wait for all our acks
+                         * best effort - just exit, user will have to kill misbehaving NCs
                          */
-                        LOGGER.info("Waiting for NCs to shutdown...");
-                        boolean cleanShutdown = shutdownStatus.waitForCompletion();
-                        if (!cleanShutdown) {
-                            /*
-                             * best effort - just exit, user will have to kill misbehaving
NCs
-                             */
-                            LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive
nodes: " +
-                                    shutdownStatus.getRemainingNodes());
-                        }
-                        callback.setValue(cleanShutdown);
-                        ccs.stop(terminateNCService);
-                        LOGGER.info("JVM Exiting.. Bye!");
-                        Runtime rt = Runtime.getRuntime();
-                        rt.exit(cleanShutdown ? 0 : 1);
-                    } catch (Exception e) {
-                        callback.setException(e);
+                        LOGGER.severe("Clean shutdown of NCs timed out- giving up; unresponsive
nodes: " +
+                                shutdownStatus.getRemainingNodes());
                     }
+                    callback.setValue(cleanShutdown);
+                    ccs.stop(terminateNCService);
+                    ExitUtil.exit(cleanShutdown ? 0 : 1);
+                } catch (Exception e) {
+                    callback.setException(e);
                 }
             });
         } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index 548ed31..b5e96e3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -54,6 +54,11 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-util</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 5601f9c..b52675c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -89,6 +89,7 @@ import org.apache.hyracks.ipc.exceptions.IPCException;
 import org.apache.hyracks.ipc.impl.IPCSystem;
 import org.apache.hyracks.net.protocols.muxdemux.FullFrameChannelInterfaceFactory;
 import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
+import org.apache.hyracks.util.ExitUtil;
 import org.kohsuke.args4j.CmdLineException;
 
 public class NodeControllerService implements IControllerService {
@@ -166,6 +167,10 @@ public class NodeControllerService implements IControllerService {
 
     private final AtomicLong maxJobId = new AtomicLong(-1);
 
+    static {
+        ExitUtil.init();
+    }
+
     public NodeControllerService(NCConfig config) throws Exception {
         this(config, getApplication(config));
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
index c3aa5f4..e9cf3cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ShutdownTask.java
@@ -24,6 +24,7 @@ import java.util.logging.Logger;
 
 import org.apache.hyracks.control.common.base.IClusterController;
 import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.util.ExitUtil;
 
 public class ShutdownTask implements Runnable {
     private static final Logger LOGGER = Logger.getLogger(ShutdownTask.class.getName());
@@ -46,15 +47,7 @@ public class ShutdownTask implements Runnable {
             // proceed with shutdown
         }
 
-        //run the shutdown in a new thread, so we don't block this last work task
-        Thread t = new Thread("NC " + ncs.getId() + " Shutdown") {
-            @Override
-            public void run() {
-                LOGGER.info("JVM Exiting.. Bye!");
-                Runtime.getRuntime().exit(terminateNCService ? 99 : 0);
-            }
-        };
-        t.start();
+        ExitUtil.exit(terminateNCService ? 99 : 0);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d2f4fbf9/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
new file mode 100644
index 0000000..b039227
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.util.logging.Logger;
+
+@SuppressWarnings("squid:S1147")
+public class ExitUtil {
+
+    private static final Logger LOGGER = Logger.getLogger(ExitUtil.class.getName());
+
+    private static final ExitThread exitThread = new ExitThread();
+
+    private ExitUtil() {
+    }
+
+    public static void init() {
+        // no-op, the clinit does the work
+    }
+
+    public static void exit(int status) {
+        exitThread.setStatus(status);
+        exitThread.start();
+    }
+
+    private static class ExitThread extends Thread {
+        private int status;
+
+        ExitThread() {
+            super("JVM exit thread");
+            setDaemon(true);
+        }
+
+        @Override
+        public void run() {
+            try {
+                LOGGER.info("JVM exiting with status " + status + "; bye!");
+            } finally {
+                Runtime.getRuntime().exit(status);
+            }
+        }
+
+        public void setStatus(int status) {
+            this.status = status;
+        }
+    }
+}


Mime
View raw message