Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7579D200B80 for ; Wed, 31 Aug 2016 05:50:06 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 73E59160AC5; Wed, 31 Aug 2016 03:50:06 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 9180C160ABA for ; Wed, 31 Aug 2016 05:50:05 +0200 (CEST) Received: (qmail 90547 invoked by uid 500); 31 Aug 2016 03:50:04 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 90537 invoked by uid 99); 31 Aug 2016 03:50:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 31 Aug 2016 03:50:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A8246E00C4; Wed, 31 Aug 2016 03:50:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mblow@apache.org To: commits@asterixdb.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: asterixdb git commit: Fix Hang At CC Shutdown When NC Has Exited Date: Wed, 31 Aug 2016 03:50:04 +0000 (UTC) archived-at: Wed, 31 Aug 2016 03:50:06 -0000 Repository: asterixdb Updated Branches: refs/heads/master 85a2e39dd -> 3954de1ad Fix Hang At CC Shutdown When NC Has Exited Also fix a few mispellings in log mesages: /UNSABLE/UNUSABLE/ /Recieved/Received/ Change-Id: I6892cfb7b7253b1e0900abaa21705ee06da477d0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1130 Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3954de1a Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3954de1a Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3954de1a Branch: refs/heads/master Commit: 3954de1ad4c5cd3d6ac928663bf15a4a47aacaea Parents: 85a2e39 Author: Michael Blow Authored: Tue Aug 30 21:47:06 2016 -0400 Committer: Michael Blow Committed: Tue Aug 30 20:49:38 2016 -0700 ---------------------------------------------------------------------- .../om/util/AsterixClusterProperties.java | 2 +- .../control/cc/work/ClusterShutdownWork.java | 21 ++++++++++++++------ .../control/cc/work/NotifyShutdownWork.java | 5 +---- .../control/common/base/INodeController.java | 2 +- .../common/ipc/NodeControllerRemoteProxy.java | 2 +- 5 files changed, 19 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3954de1a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java index abfcf67..5acf9ae 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixClusterProperties.java @@ -205,7 +205,7 @@ public class AsterixClusterProperties { for (ClusterPartition p : clusterPartitions.values()) { if (!p.isActive()) { state = ClusterState.UNUSABLE; - LOGGER.info("Cluster is in UNSABLE state"); + LOGGER.info("Cluster is in UNUSABLE state"); return; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3954de1a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java index b05a3d6..6acab12 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ClusterShutdownWork.java @@ -22,6 +22,7 @@ package org.apache.hyracks.control.cc.work; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.hyracks.control.cc.ClusterControllerService; @@ -60,9 +61,8 @@ public class ClusterShutdownWork extends SynchronizableWork { /** * Shutdown all the nodes... */ - for (NodeControllerState ncs : nodeControllerStateMap.values()) { - ncs.getNodeController().shutDown(); - } + nodeControllerStateMap.forEach(this::shutdownNode); + ccs.getExecutor().execute(new Runnable() { @Override public void run() { @@ -72,7 +72,7 @@ public class ClusterShutdownWork extends SynchronizableWork { */ boolean cleanShutdown = shutdownStatus.waitForCompletion(); if (cleanShutdown) { - callback.setValue(new Boolean(true)); + callback.setValue(true); ccs.stop(); LOGGER.info("JVM Exiting.. Bye!"); Runtime rt = Runtime.getRuntime(); @@ -85,10 +85,10 @@ public class ClusterShutdownWork extends SynchronizableWork { LOGGER.severe("Clean shutdown of NCs timed out- CC bailing out!"); StringBuilder unresponsive = new StringBuilder(); for (String s : shutdownStatus.getRemainingNodes()) { - unresponsive.append(s + " "); + unresponsive.append(s).append(' '); } LOGGER.severe("Unresponsive Nodes: " + unresponsive); - callback.setValue(new Boolean(false)); + callback.setValue(false); ccs.stop(); LOGGER.info("JVM Exiting.. Bye!"); Runtime rt = Runtime.getRuntime(); @@ -103,4 +103,13 @@ public class ClusterShutdownWork extends SynchronizableWork { callback.setException(e); } } + + protected void shutdownNode(String key, NodeControllerState ncState) { + try { + ncState.getNodeController().shutdown(); + } catch (Exception e) { + LOGGER.log( + Level.INFO, "Exception shutting down NC " + key + " (possibly dead?), continuing shutdown...", e); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3954de1a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java index 9e290d6..1efbc6a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyShutdownWork.java @@ -21,10 +21,7 @@ package org.apache.hyracks.control.cc.work; import java.util.logging.Logger; -import org.apache.hyracks.api.deployment.DeploymentId; import org.apache.hyracks.control.cc.ClusterControllerService; -import org.apache.hyracks.control.common.deployment.DeploymentRun; -import org.apache.hyracks.control.common.deployment.DeploymentStatus; import org.apache.hyracks.control.common.shutdown.ShutdownRun; import org.apache.hyracks.control.common.work.SynchronizableWork; @@ -44,7 +41,7 @@ public class NotifyShutdownWork extends SynchronizableWork { public void doRun() { /** triggered remotely by a NC to notify that the NC is shutting down */ ShutdownRun sRun = ccs.getShutdownRun(); - LOGGER.info("Recieved shutdown acknowledgement from NC ID:" + nodeId); + LOGGER.info("Received shutdown acknowledgement from NC ID:" + nodeId); sRun.notifyShutdown(nodeId); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3954de1a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java index 7fe2f58..75c3127 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/INodeController.java @@ -51,7 +51,7 @@ public interface INodeController { public void dumpState(String stateDumpId) throws Exception; - public void shutDown() throws Exception; + public void shutdown() throws Exception; public void sendApplicationMessageToNC(byte[] data, DeploymentId deploymentId, String nodeId) throws Exception; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3954de1a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java index f7f73d7..841c889 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/NodeControllerRemoteProxy.java @@ -90,7 +90,7 @@ public class NodeControllerRemoteProxy implements INodeController { } @Override - public void shutDown() throws Exception { + public void shutdown() throws Exception { CCNCFunctions.ShutdownRequestFunction sdrf = new CCNCFunctions.ShutdownRequestFunction(); ipcHandle.send(-1, sdrf, null); }