asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [4/6] asterixdb git commit: [NO ISSUE][CLUS] Asynchronous reregistration with CC
Date Thu, 22 Mar 2018 01:11:29 GMT
[NO ISSUE][CLUS] Asynchronous reregistration with CC

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Perform CC reregistration asynchronously on a
  separate thread when connection is restored.
- Ensure a single registration is sent to CC when
  connection is restored.
- Restart JVM on unexpected CC registration request
  failures.
- Halt JVM on registration failures or timeout.

Change-Id: I404256e6f550c42a6eaf17c0ae4defb7ffb9cb2f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2505
Reviewed-by: Michael Blow <mblow@apache.org>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/01d7fe9a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/01d7fe9a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/01d7fe9a

Branch: refs/heads/master
Commit: 01d7fe9aee2b21481abc6cd896bf873e5d8f8df4
Parents: 4d0de7d
Author: Murtadha Hubail <mhubail@apache.org>
Authored: Wed Mar 21 05:13:09 2018 +0300
Committer: Murtadha Hubail <mhubail@apache.org>
Committed: Tue Mar 20 21:34:05 2018 -0700

----------------------------------------------------------------------
 .../apache/hyracks/control/nc/CcConnection.java | 39 ++++++++++++++++++--
 .../control/nc/NodeControllerService.java       | 11 +++---
 2 files changed, 41 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/01d7fe9a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
index 1c6c98e..dce7d35 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
@@ -18,10 +18,15 @@
  */
 package org.apache.hyracks.control.nc;
 
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.hyracks.api.control.CcId;
+import org.apache.hyracks.api.util.InvokeUtil;
 import org.apache.hyracks.control.common.base.IClusterController;
 import org.apache.hyracks.control.common.controllers.NodeParameters;
 import org.apache.hyracks.control.common.controllers.NodeRegistration;
+import org.apache.hyracks.util.ExitUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -31,6 +36,7 @@ public class CcConnection {
 
     private final IClusterController ccs;
     private boolean registrationPending;
+    private boolean registrationCompleted;
     private Exception registrationException;
     private NodeParameters nodeParameters;
 
@@ -57,12 +63,14 @@ public class CcConnection {
     public synchronized CcId registerNode(NodeRegistration nodeRegistration, int registrationId)
throws Exception {
         registrationPending = true;
         ccs.registerNode(nodeRegistration, registrationId);
-        while (registrationPending) {
-            wait();
+        try {
+            InvokeUtil.runWithTimeout(this::wait, () -> !registrationPending, 2, TimeUnit.MINUTES);
+        } catch (Exception e) {
+            registrationException = e;
         }
         if (registrationException != null) {
-            LOGGER.log(Level.WARN, "Registering with {} failed with exception", this, registrationException);
-            throw registrationException;
+            LOGGER.fatal("Registering with {} failed with exception", this, registrationException);
+            ExitUtil.halt(ExitUtil.EC_IMMEDIATE_HALT);
         }
         return getCcId();
     }
@@ -74,4 +82,27 @@ public class CcConnection {
     public NodeParameters getNodeParameters() {
         return nodeParameters;
     }
+
+    public synchronized void notifyConnectionRestored(NodeControllerService ncs, InetSocketAddress
ccAddress)
+            throws InterruptedException {
+        if (registrationCompleted) {
+            registrationCompleted = false;
+            ncs.getExecutor().submit(() -> {
+                try {
+                    return ncs.registerNode(this, ccAddress);
+                } catch (Exception e) {
+                    LOGGER.log(Level.ERROR, "Failed registering with cc", e);
+                    throw new IllegalStateException(e);
+                }
+            });
+        }
+        while (!registrationCompleted) {
+            wait();
+        }
+    }
+
+    public synchronized void notifyRegistrationCompleted() {
+        registrationCompleted = true;
+        notifyAll();
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/01d7fe9a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 0756210..98f5c70 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -352,10 +352,11 @@ public class NodeControllerService implements IControllerService {
                 @Override
                 public void ipcHandleRestored(IIPCHandle handle) throws IPCException {
                     // we need to re-register in case of NC -> CC connection reset
+                    final CcConnection ccConnection = getCcConnection(ccAddressMap.get(ccAddress));
                     try {
-                        registerNode(getCcConnection(ccAddressMap.get(ccAddress)), ccAddress);
-                    } catch (Exception e) {
-                        LOGGER.log(Level.WARN, "Failed Registering with cc", e);
+                        ccConnection.notifyConnectionRestored(NodeControllerService.this,
ccAddress);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
                         throw new IPCException(e);
                     }
                 }
@@ -412,7 +413,7 @@ public class NodeControllerService implements IControllerService {
         }
     }
 
-    private CcId registerNode(CcConnection ccc, InetSocketAddress ccAddress) throws Exception
{
+    public CcId registerNode(CcConnection ccc, InetSocketAddress ccAddress) throws Exception
{
         LOGGER.info("Registering with Cluster Controller {}", ccc);
 
         int registrationId = nextRegistrationId.incrementAndGet();
@@ -444,7 +445,7 @@ public class NodeControllerService implements IControllerService {
             ccTimer.schedule(new ProfileDumpTask(ccs, ccId), 0, nodeParameters.getProfileDumpPeriod());
             ccTimers.put(ccId, ccTimer);
         }
-
+        ccc.notifyRegistrationCompleted();
         LOGGER.info("Registering with Cluster Controller {} complete", ccc);
         return ccId;
     }


Mime
View raw message