brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [07/21] incubator-brooklyn git commit: Add a HOT_BACKUP mode where nodes are read-only copies but NOT willing to be master.
Date Sat, 15 Nov 2014 00:05:16 GMT
Add a HOT_BACKUP mode where nodes are read-only copies but NOT willing to be master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/9dd1a957
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/9dd1a957
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/9dd1a957

Branch: refs/heads/master
Commit: 9dd1a95724a473b59a777fb6d627b6dde2fa0547
Parents: f39c6a3
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Thu Nov 13 02:05:16 2014 +0000
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Thu Nov 13 23:38:53 2014 +0000

----------------------------------------------------------------------
 .../brooklyn/entity/rebind/RebindManager.java   |   6 +-
 .../management/ha/HighAvailabilityManager.java  |   2 +-
 .../management/ha/HighAvailabilityMode.java     |   6 +
 .../management/ha/ManagementNodeState.java      |  25 +++-
 .../entity/rebind/RebindManagerImpl.java        |  22 +--
 .../ha/HighAvailabilityManagerImpl.java         | 140 +++++++++++--------
 .../ha/ManagementPlaneSyncRecordDeltaImpl.java  |   1 -
 ...mentPlaneSyncRecordPersisterToMultiFile.java |   4 -
 .../NonDeploymentManagementContext.java         |   2 +-
 .../entity/rebind/RebindManagerSorterTest.java  |   2 +-
 usage/cli/src/main/java/brooklyn/cli/Main.java  |  18 ++-
 .../brooklyn/launcher/BrooklynLauncher.java     |  20 +--
 12 files changed, 154 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/entity/rebind/RebindManager.java b/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
index e5001a6..fcd71bc 100644
--- a/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
+++ b/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
@@ -69,7 +69,7 @@ public interface RebindManager {
     /** Causes this management context to rebind, loading data from the given backing store.
      * use wisely, as this can cause local entities to be completely lost, or will throw
in many other situations.
      * in general it may be invoked for a new node becoming {@link ManagementNodeState#MASTER}

-     * or periodically for a node in {@link ManagementNodeState#HOT_STANDBY}. */
+     * or periodically for a node in {@link ManagementNodeState#HOT_STANDBY} or {@link ManagementNodeState#HOT_BACKUP}.
*/
     @Beta
     public List<Application> rebind(ClassLoader classLoader, RebindExceptionHandler
exceptionHandler, ManagementNodeState mode);
 
@@ -92,13 +92,13 @@ public interface RebindManager {
      * Perform an initial load of state read-only and starts a background process 
      * reading (mirroring) state periodically.
      */
-    public void startReadOnly();
+    public void startReadOnly(ManagementNodeState mode);
     /** Stops the background reading (mirroring) of state. 
      * Interrupts any current activity and waits for it to cease. */
     public void stopReadOnly();
     
     /** Starts the appropriate background processes, {@link #startPersistence()} if {@link
ManagementNodeState#MASTER},
-     * {@link #startReadOnly()} if {@link ManagementNodeState#HOT_STANDBY} */
+     * {@link #startReadOnly()} if {@link ManagementNodeState#HOT_STANDBY} or {@link ManagementNodeState#HOT_BACKUP}
*/
     public void start();
     /** Stops the appropriate background processes, {@link #stopPersistence()} or {@link
#stopReadOnly()},
      * waiting for activity there to cease (interrupting in the case of {@link #stopReadOnly()}).
*/

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
index c3671d7..de37123 100644
--- a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
+++ b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
@@ -72,7 +72,7 @@ public interface HighAvailabilityManager {
      * <p>
      * When this method returns, the status of this node will be set,
      * either {@link ManagementNodeState#MASTER} if appropriate 
-     * or {@link ManagementNodeState#STANDBY} / {@link ManagementNodeState#HOT_STANDBY}.
+     * or {@link ManagementNodeState#STANDBY} / {@link ManagementNodeState#HOT_STANDBY} /
{@link ManagementNodeState#HOT_BACKUP}.
      *
      * @param startMode mode to start with
      * @throws IllegalStateException if current state of the management-plane doesn't match
that desired by {@code startMode} 

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/api/src/main/java/brooklyn/management/ha/HighAvailabilityMode.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/management/ha/HighAvailabilityMode.java b/api/src/main/java/brooklyn/management/ha/HighAvailabilityMode.java
index 97fb5bc..2e593b1 100644
--- a/api/src/main/java/brooklyn/management/ha/HighAvailabilityMode.java
+++ b/api/src/main/java/brooklyn/management/ha/HighAvailabilityMode.java
@@ -52,6 +52,12 @@ public enum HighAvailabilityMode {
     HOT_STANDBY,
     
     /**
+     * Means node must be hot backup; do not attempt to become master (but it <i>can</i>
start without a master).
+     * See {@link ManagementNodeState#HOT_BACKUP}. 
+     */
+    HOT_BACKUP,
+    
+    /**
      * Means node must be master; if there is already a master then fail fast on startup.
      * See {@link ManagementNodeState#MASTER}.
      */

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/api/src/main/java/brooklyn/management/ha/ManagementNodeState.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/management/ha/ManagementNodeState.java b/api/src/main/java/brooklyn/management/ha/ManagementNodeState.java
index c8a90f1..4587b57 100644
--- a/api/src/main/java/brooklyn/management/ha/ManagementNodeState.java
+++ b/api/src/main/java/brooklyn/management/ha/ManagementNodeState.java
@@ -18,6 +18,8 @@
  */
 package brooklyn.management.ha;
 
+import brooklyn.util.guava.Maybe;
+
 public enum ManagementNodeState {
     /** @deprecated since 0.7.0 synonym for maintenance (plus, it should have been US English!)
*/
     UNINITIALISED,
@@ -27,8 +29,10 @@ public enum ManagementNodeState {
     /** Node is in "lukewarm standby" mode, where it is available to be promoted to master,
      * but does not have entities loaded and will require some effort to be promoted */
     STANDBY,
-    /** Node is acting as read-only proxy */
+    /** Node is acting as read-only proxy available to be promoted to master on existing
master failure */
     HOT_STANDBY,
+    /** Node is acting as a read-only proxy but not making itself available for promotion
to master */
+    HOT_BACKUP,
     /** Node is running as primary/master, able to manage entities and create new ones */
     // the semantics are intended to support multi-master here; we could have multiple master
nodes,
     // but we need to look up who is master for any given entity
@@ -38,4 +42,23 @@ public enum ManagementNodeState {
     FAILED,
     /** Node has gone away; maintenance not possible */
     TERMINATED;
+
+    /** Converts a {@link HighAvailabilityMode} to a {@link ManagementNodeState}, if possible
*/
+    public static Maybe<ManagementNodeState> of(HighAvailabilityMode startMode) {
+        switch (startMode) {
+        case AUTO:
+        case DISABLED:
+            return Maybe.absent("Requested "+HighAvailabilityMode.class+" mode "+startMode+"
cannot be converted to "+ManagementNodeState.class);
+        case HOT_BACKUP:
+            return Maybe.of(HOT_BACKUP);
+        case HOT_STANDBY:
+            return Maybe.of(HOT_STANDBY);
+        case MASTER:
+            return Maybe.of(MASTER);
+        case STANDBY:
+            return Maybe.of(STANDBY);
+        }
+        // above should be exhaustive
+        return Maybe.absent("Requested "+HighAvailabilityMode.class+" mode "+startMode+"
was not expected");
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
index ac9f726..4252a25 100644
--- a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
+++ b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
@@ -298,7 +298,11 @@ public class RebindManagerImpl implements RebindManager {
     
     @SuppressWarnings("unchecked")
     @Override
-    public void startReadOnly() {
+    public void startReadOnly(final ManagementNodeState mode) {
+        if (mode!=ManagementNodeState.HOT_STANDBY && mode!=ManagementNodeState.HOT_BACKUP)
{
+            throw new IllegalStateException("Read-only rebind thread only permitted for hot
proxy modes; not "+mode);
+        }
+        
         if (persistenceRunning) {
             throw new IllegalStateException("Cannot start read-only when already running
with persistence");
         }
@@ -315,7 +319,7 @@ public class RebindManagerImpl implements RebindManager {
         readOnlyRebindCount = 0;
 
         try {
-            rebind(null, null, ManagementNodeState.HOT_STANDBY);
+            rebind(null, null, mode);
         } catch (Exception e) {
             Exceptions.propagate(e);
         }
@@ -325,7 +329,7 @@ public class RebindManagerImpl implements RebindManager {
                 return Tasks.<Void>builder().dynamic(false).name("rebind (periodic
run").body(new Callable<Void>() {
                     public Void call() {
                         try {
-                            rebind(null, null, ManagementNodeState.HOT_STANDBY);
+                            rebind(null, null, mode);
                             readOnlyRebindCount++;
                             return null;
                         } catch (RuntimeInterruptedException e) {
@@ -376,8 +380,8 @@ public class RebindManagerImpl implements RebindManager {
     @Override
     public void start() {
         ManagementNodeState target = getRebindMode();
-        if (target==ManagementNodeState.HOT_STANDBY) {
-            startReadOnly();
+        if (target==ManagementNodeState.HOT_STANDBY || target==ManagementNodeState.HOT_BACKUP)
{
+            startReadOnly(target);
         } else if (target==ManagementNodeState.MASTER) {
             startPersistence();
         } else {
@@ -466,8 +470,8 @@ public class RebindManagerImpl implements RebindManager {
                 .build();
         final ManagementNodeState mode = modeO!=null ? modeO : getRebindMode();
         
-        if (mode!=ManagementNodeState.HOT_STANDBY && mode!=ManagementNodeState.MASTER)
-            throw new IllegalStateException("Must be either master or read only to rebind
(mode "+mode+")");
+        if (mode!=ManagementNodeState.MASTER && mode!=ManagementNodeState.HOT_STANDBY
&& mode!=ManagementNodeState.HOT_BACKUP)
+            throw new IllegalStateException("Must be either master or hot standby/backup
to rebind (mode "+mode+")");
 
         ExecutionContext ec = BasicExecutionContext.getCurrentExecutionContext();
         if (ec == null) {
@@ -533,7 +537,7 @@ public class RebindManagerImpl implements RebindManager {
             
             exceptionHandler.onStart(rebindContext);
             
-            if (mode==ManagementNodeState.HOT_STANDBY) {
+            if (mode==ManagementNodeState.HOT_STANDBY || mode==ManagementNodeState.HOT_BACKUP)
{
                 rebindContext.setAllReadOnly();
             } else {
                 Preconditions.checkState(mode==ManagementNodeState.MASTER, "Must be either
master or read only to rebind (mode "+mode+")");
@@ -590,7 +594,7 @@ public class RebindManagerImpl implements RebindManager {
             BrooklynMementoManifest mementoManifest = persistenceStoreAccess.loadMementoManifest(mementoRawData,
exceptionHandler);
 
             boolean isEmpty = mementoManifest.isEmpty();
-            if (mode!=ManagementNodeState.HOT_STANDBY) {
+            if (mode!=ManagementNodeState.HOT_STANDBY && mode!=ManagementNodeState.HOT_BACKUP)
{
                 if (!isEmpty) { 
                     LOG.info("Rebinding from "+getPersister().getBackingStoreDescription()+"...");
                 } else {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
index 329578e..62b06c0 100644
--- a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
+++ b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
@@ -98,20 +98,8 @@ import com.google.common.collect.Iterables;
 @Beta
 public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
 
-    // TODO Improve mechanism for injecting configuration options (such as heartbeat and
timeouts).
-    // For example, read from brooklyn.properties?
-    // But see BrooklynLauncher.haHeartbeatPeriod and .haHeartbeatTimeout, which are always
injected.
-    // So perhaps best to read brooklyn.properties there? Would be nice to avoid the cast
to 
-    // HighAvailabilityManagerImpl though.
-    
-    // TODO There is a race if you start multiple nodes simultaneously.
-    // They may not have seen each other's heartbeats yet, so will all claim mastery!
-    // But this should be resolved shortly afterwards.
-
-    // TODO Should we pass in a classloader on construction, so it can be passed to {@link
RebindManager#rebind(ClassLoader)}
-
     public final ConfigKey<Duration> POLL_PERIOD = ConfigKeys.newConfigKey(Duration.class,
"brooklyn.ha.pollPeriod",
-        "How often nodes should poll to detect whether master is healthy", Duration.seconds(5));
+        "How often nodes should poll to detect whether master is healthy", Duration.seconds(1));
     public final ConfigKey<Duration> HEARTBEAT_TIMEOUT = ConfigKeys.newConfigKey(Duration.class,
"brooklyn.ha.heartbeatTimeout",
         "Maximum allowable time for detection of a peer's heartbeat; if no sign of master
after this time, "
         + "another node may promote itself", Duration.THIRTY_SECONDS);
@@ -286,8 +274,10 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             case DISABLED:
                 // no action needed, will do anything necessary below
                 break;
-            case HOT_STANDBY: demoteToStandby(true); break;
-            case STANDBY: demoteToStandby(false); break;
+            case HOT_STANDBY: 
+            case HOT_BACKUP: 
+            case STANDBY: 
+                demoteTo(ManagementNodeState.of(startMode).get()); break;
             default:
                 throw new IllegalStateException("Unexpected high availability mode "+startMode+"
requested for "+this);
             }
@@ -298,8 +288,14 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         case AUTO:
             // don't care; let's start and see if we promote ourselves
             publishAndCheck(true);
-            if (nodeState == ManagementNodeState.STANDBY || nodeState == ManagementNodeState.HOT_STANDBY)
{
-                ManagementPlaneSyncRecord newState = loadManagementPlaneSyncRecord(true);;
+            switch (nodeState) {
+            case HOT_BACKUP:
+                if (!nodeStateTransitionComplete) throw new IllegalStateException("Cannot
switch to AUTO when in the middle of a transition to "+nodeState);
+                // else change us to hot standby and continue to below
+                nodeState = ManagementNodeState.HOT_STANDBY;
+            case HOT_STANDBY:
+            case STANDBY:
+                ManagementPlaneSyncRecord newState = loadManagementPlaneSyncRecord(true);
                 String masterNodeId = newState.getMasterNodeId();
                 ManagementNodeSyncRecord masterNodeDetails = newState.getManagementNodes().get(masterNodeId);
                 LOG.info("Management node "+ownNodeId+" running as HA " + nodeState + " autodetected,
" +
@@ -307,9 +303,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                         + (existingMaster==null ? "(new) " : "")
                         + "is "+masterNodeId +
                         (masterNodeDetails==null || masterNodeDetails.getUri()==null ? "
(no url)" : " at "+masterNodeDetails.getUri())));
-            } else if (nodeState == ManagementNodeState.MASTER) {
+                break;
+            case MASTER:
                 LOG.info("Management node "+ownNodeId+" running as HA MASTER autodetected");
-            } else {
+                break;
+            default:
                 throw new IllegalStateException("Management node "+ownNodeId+" set to HA
AUTO, encountered unexpected mode "+nodeState);
             }
             break;
@@ -332,7 +330,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                 LOG.error("Management node "+ownNodeId+" detected no master when "+startMode+"
requested and existing master required; failing.");
                 throw new IllegalStateException("No existing master; cannot start as "+startMode);
             }
-            
+            // continue to below (above lines skipped for hot backup)
+        case HOT_BACKUP:
             String message = "Management node "+ownNodeId+" running as HA "+getNodeState()+"
(";
             if (getNodeState().toString().equals(startMode.toString()))
                 message += "explicitly requested";
@@ -346,7 +345,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             } else {
                 ManagementPlaneSyncRecord newState = loadManagementPlaneSyncRecord(true);
                 if (Strings.isBlank(newState.getMasterNodeId())) {
-                    message += "); no master currently (subsequent election may repair)";
+                    message += "); no master currently"; 
+                    if (startMode != HighAvailabilityMode.HOT_BACKUP) message += " (subsequent
election may repair)";
                 } else {
                     message += "); master "+newState.getMasterNodeId();
                 }
@@ -356,7 +356,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         case DISABLED:
             // safe just to run even if we weren't master
             LOG.info("Management node "+ownNodeId+" HA DISABLED (was "+nodeState+")");
-            demoteToFailed();
+            demoteTo(ManagementNodeState.FAILED);
             if (pollingTask!=null) pollingTask.cancel(true);
             break;
         default:
@@ -370,25 +370,28 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                 startMode = HighAvailabilityMode.STANDBY;
             }
         }
-        if (nodeState==ManagementNodeState.STANDBY && startMode==HighAvailabilityMode.HOT_STANDBY)
{
-            // if it should be hot standby, then we need to promote
+        if ((nodeState==ManagementNodeState.STANDBY && startMode==HighAvailabilityMode.HOT_STANDBY)
|| 
+                (startMode==HighAvailabilityMode.HOT_BACKUP)) {
             nodeStateTransitionComplete = false;
-            // inform the world that we are transitioning (not eligible for promotion while
going in to hot standby)
-            publishHealth();
+            if (startMode==HighAvailabilityMode.HOT_STANDBY) {
+                // if it should be hot standby, then we need to promote
+                // inform the world that we are transitioning (not eligible for promotion
while going in to hot standby)
+                publishHealth();
+            }
             try {
-                attemptHotStandby();
+                attemptHotProxy(ManagementNodeState.of(startMode).get());
                 nodeStateTransitionComplete = true;
                 publishHealth();
                 
-                if (getNodeState()==ManagementNodeState.HOT_STANDBY) {
-                    LOG.info("Management node "+ownNodeId+" now running as HA "+ManagementNodeState.HOT_STANDBY+";
"
+                if (getNodeState()==ManagementNodeState.HOT_STANDBY || getNodeState()==ManagementNodeState.HOT_BACKUP)
{
+                    LOG.info("Management node "+ownNodeId+" now running as HA "+getNodeState()+";
"
                         + managementContext.getApplications().size()+" application"+Strings.s(managementContext.getApplications().size())+"
loaded");
                 } else {
-                    LOG.warn("Management node "+ownNodeId+" unable to promote to "+ManagementNodeState.HOT_STANDBY+"
(currently "+getNodeState()+"); "
+                    LOG.warn("Management node "+ownNodeId+" unable to promote to "+startMode+"
(currently "+getNodeState()+"); "
                         + "(see log for further details)");
                 }
             } catch (Exception e) {
-                LOG.warn("Management node "+ownNodeId+" unable to promote to "+ManagementNodeState.HOT_STANDBY+"
(currently "+getNodeState()+"); rethrowing: "+Exceptions.collapseText(e));
+                LOG.warn("Management node "+ownNodeId+" unable to promote to "+startMode+"
(currently "+getNodeState()+"); rethrowing: "+Exceptions.collapseText(e));
                 throw Exceptions.propagate(e);
             }
         } else {
@@ -594,9 +597,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
     protected void checkMaster(boolean initializing) {
         ManagementPlaneSyncRecord memento = loadManagementPlaneSyncRecord(false);
         
-        if (getNodeState() == ManagementNodeState.FAILED) {
-            // if we have failed then no point in checking who is master
-            // (if somehow this node is subsequently clearFailure() then it will resume)
+        if (getNodeState()==ManagementNodeState.FAILED || getNodeState()==ManagementNodeState.HOT_BACKUP)
{
+            // if failed or hot backup then we can't promote ourselves, so no point in checking
who is master
             return;
         }
         
@@ -642,7 +644,9 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         
         if (demotingSelfInFavourOfOtherMaster) {
             LOG.debug("Master-change for this node only, demoting "+ownNodeRecord.toVerboseString()+"
in favour of official master "+newMasterNodeRecord.toVerboseString());
-            demoteToStandby(BrooklynFeatureEnablement.isEnabled(BrooklynFeatureEnablement.FEATURE_DEFAULT_STANDBY_IS_HOT_PROPERTY));
+            demoteTo(
+                BrooklynFeatureEnablement.isEnabled(BrooklynFeatureEnablement.FEATURE_DEFAULT_STANDBY_IS_HOT_PROPERTY)
?
+                    ManagementNodeState.HOT_STANDBY : ManagementNodeState.STANDBY);
             return;
         } else {
             LOG.debug("Detected master heartbeat timeout. Initiating a new master election.
Master was " + currMasterNodeRecord);
@@ -703,11 +707,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                 LOG.warn("Problem in promption-listener (continuing)", e);
             }
         }
-        boolean wasHotStandby = nodeState==ManagementNodeState.HOT_STANDBY;
+        boolean wasHot = (nodeState==ManagementNodeState.HOT_STANDBY || nodeState==ManagementNodeState.HOT_BACKUP);
         nodeState = ManagementNodeState.MASTER;
         publishPromotionToMaster();
         try {
-            if (wasHotStandby) {
+            if (wasHot) {
                 // could just promote the standby items; but for now we stop the old read-only
and re-load them, to make sure nothing has been missed
                 // TODO ideally there'd be an incremental rebind as well as an incremental
persist
                 managementContext.getRebindManager().stopReadOnly();
@@ -716,7 +720,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             managementContext.getRebindManager().rebind(managementContext.getCatalog().getRootClassLoader(),
null, nodeState);
         } catch (Exception e) {
             LOG.error("Management node enountered problem during rebind when promoting self
to master; demoting to FAILED and rethrowing: "+e);
-            demoteToFailed();
+            demoteTo(ManagementNodeState.FAILED);
             throw Exceptions.propagate(e);
         }
         managementContext.getRebindManager().start();
@@ -728,19 +732,17 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         }
     }
 
+    /** @deprecated since 0.7.0, use {@link #demoteTo(ManagementNodeState)} */ @Deprecated
     protected void demoteToFailed() {
-        // TODO merge this method with the one below
-        boolean wasMaster = nodeState == ManagementNodeState.MASTER;
-        if (wasMaster) backupOnDemotionIfNeeded();
-        ManagementTransitionMode mode = (wasMaster ? ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY
: ManagementTransitionMode.REBINDING_DESTROYED);
-        nodeState = ManagementNodeState.FAILED;
-        onDemotionStopItems(mode);
-        nodeStateTransitionComplete = true;
-        publishDemotion(wasMaster);
+        demoteTo(ManagementNodeState.FAILED);
     }
-    
+    /** @deprecated since 0.7.0, use {@link #demoteTo(ManagementNodeState)} */ @Deprecated
     protected void demoteToStandby(boolean hot) {
-        if (!running) {
+        demoteTo(hot ? ManagementNodeState.HOT_STANDBY : ManagementNodeState.STANDBY);
+    }
+    
+    protected void demoteTo(ManagementNodeState toState) {
+        if (toState!=ManagementNodeState.FAILED && !running) {
             LOG.warn("Ignoring demote-from-master request, as HighAvailabilityManager is
no longer running");
             return;
         }
@@ -749,14 +751,24 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         ManagementTransitionMode mode = (wasMaster ? ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY
: ManagementTransitionMode.REBINDING_DESTROYED);
 
         nodeStateTransitionComplete = false;
-        nodeState = ManagementNodeState.STANDBY;
+        
+        switch (toState) {
+        case FAILED: 
+        case HOT_BACKUP:
+        case STANDBY:
+            nodeState = toState; break;
+        case HOT_STANDBY:
+            nodeState = ManagementNodeState.STANDBY; break;
+        default:
+            throw new IllegalStateException("Illegal target state: "+toState);
+        }
         onDemotionStopItems(mode);
         nodeStateTransitionComplete = true;
         publishDemotion(wasMaster);
         
-        if (hot) {
+        if (toState==ManagementNodeState.HOT_BACKUP || toState==ManagementNodeState.HOT_STANDBY)
{
             nodeStateTransitionComplete = false;
-            attemptHotStandby();
+            attemptHotProxy(toState);
             nodeStateTransitionComplete = true;
             publishHealth();
         }
@@ -795,19 +807,29 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         ((BasicBrooklynCatalog)managementContext.getCatalog()).reset(CatalogDto.newEmptyInstance("<reset-by-ha-status-change>"));
     }
     
-    /** starts hot standby, in foreground; the caller is responsible for publishing health
afterwards.
-     * @return whether hot standby was possible (if not, errors should be stored elsewhere)
*/
+    /** @deprecated since 0.7.0, use {@link #attemptHotProxy(ManagementNodeState)} */ @Deprecated
     protected boolean attemptHotStandby() {
+        return attemptHotProxy(ManagementNodeState.HOT_STANDBY);
+    }
+    
+    /** Starts hot standby or hot backup, in foreground
+     * <p>
+     * In the case of the former, the caller is responsible for publishing health afterwards,
+     * but if it fails, this method will {@link #demoteTo(ManagementNodeState)} {@link ManagementNodeState#FAILED}.
+     * <p>
+     * @return whether the requested {@link ManagementNodeState} was possible;
+     * (if not, errors should be stored elsewhere) */
+    protected boolean attemptHotProxy(ManagementNodeState toState) {
         try {
-            Preconditions.checkState(nodeStateTransitionComplete==false, "Must be in transitioning
state to go into hot standby");
-            nodeState = ManagementNodeState.HOT_STANDBY;
-            managementContext.getRebindManager().startReadOnly();
+            Preconditions.checkState(nodeStateTransitionComplete==false, "Must be in transitioning
state to go into "+toState);
+            nodeState = toState;
+            managementContext.getRebindManager().startReadOnly(toState);
             
             return true;
         } catch (Exception e) {
             Exceptions.propagateIfFatal(e);
-            LOG.warn("Unable to promote "+ownNodeId+" to hot standby, switching to FAILED:
"+e, e);
-            demoteToFailed();
+            LOG.warn("Unable to promote "+ownNodeId+" to "+toState+", switching to FAILED:
"+e, e);
+            demoteTo(ManagementNodeState.FAILED);
             return false;
         }
     }
@@ -909,7 +931,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         @Override
         public ManagementNodeSyncRecord apply(@Nullable ManagementNodeSyncRecord input) {
             if (input == null) return null;
-            if (!(input.getStatus() == ManagementNodeState.STANDBY || input.getStatus() ==
ManagementNodeState.HOT_STANDBY || input.getStatus() == ManagementNodeState.MASTER)) return
input;
+            if (!(input.getStatus() == ManagementNodeState.STANDBY || input.getStatus() ==
ManagementNodeState.HOT_STANDBY || input.getStatus() == ManagementNodeState.MASTER || input.getStatus()
== ManagementNodeState.HOT_BACKUP)) return input;
             if (isHeartbeatOk(input, referenceNode)) return input;
             return BasicManagementNodeSyncRecord.builder()
                     .from(input)

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordDeltaImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordDeltaImpl.java
b/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordDeltaImpl.java
index 6b026bc..5e07b19 100644
--- a/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordDeltaImpl.java
+++ b/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordDeltaImpl.java
@@ -25,7 +25,6 @@ import java.util.Collection;
 
 import brooklyn.management.ha.ManagementPlaneSyncRecordPersister.Delta;
 
-import com.google.api.client.repackaged.com.google.common.base.Objects;
 import com.google.common.annotations.Beta;
 import com.google.common.collect.Sets;
 

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordPersisterToMultiFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordPersisterToMultiFile.java
b/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordPersisterToMultiFile.java
index d3fa34a..fcade26 100644
--- a/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordPersisterToMultiFile.java
+++ b/core/src/main/java/brooklyn/management/ha/ManagementPlaneSyncRecordPersisterToMultiFile.java
@@ -274,10 +274,6 @@ public class ManagementPlaneSyncRecordPersisterToMultiFile implements
Management
         return new File(dir, "master");
     }
 
-    private File getFileForPlaneId() {
-        return new File(dir, "plane.id");
-    }
-
     private File getFileForChangeLog() {
         return new File(dir, "change.log");
     }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
index 3d0e4d5..2771475 100644
--- a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
+++ b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
@@ -491,7 +491,7 @@ public class NonDeploymentManagementContext implements ManagementContextInternal
         }
 
         @Override
-        public void startReadOnly() {
+        public void startReadOnly(ManagementNodeState state) {
             throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+"
is not valid for this operation.");
         }
         

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/core/src/test/java/brooklyn/entity/rebind/RebindManagerSorterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/RebindManagerSorterTest.java b/core/src/test/java/brooklyn/entity/rebind/RebindManagerSorterTest.java
index 991a000..f986c7e 100644
--- a/core/src/test/java/brooklyn/entity/rebind/RebindManagerSorterTest.java
+++ b/core/src/test/java/brooklyn/entity/rebind/RebindManagerSorterTest.java
@@ -142,7 +142,7 @@ public class RebindManagerSorterTest {
     private Map<String, EntityMemento> toMementos(Iterable<? extends Entity>
entities) {
         Map<String, EntityMemento> result = Maps.newLinkedHashMap();
         for (Entity entity : entities) {
-            result.put(entity.getId(), MementosGenerators.newEntityMemento(entity));
+            result.put(entity.getId(), MementosGenerators.newEntityMemento(Entities.deproxy(entity)));
         }
         return result;
     }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/usage/cli/src/main/java/brooklyn/cli/Main.java
----------------------------------------------------------------------
diff --git a/usage/cli/src/main/java/brooklyn/cli/Main.java b/usage/cli/src/main/java/brooklyn/cli/Main.java
index b1b933c..3eb7fc0 100644
--- a/usage/cli/src/main/java/brooklyn/cli/Main.java
+++ b/usage/cli/src/main/java/brooklyn/cli/Main.java
@@ -313,17 +313,22 @@ public class Main extends AbstractMain {
         protected final static String HA_OPTION_MASTER = "master";
         protected final static String HA_OPTION_STANDBY = "standby";
         protected final static String HA_OPTION_HOT_STANDBY = "hot_standby";
-        static { Enums.checkAllEnumeratedIgnoreCase(HighAvailabilityMode.class, HA_OPTION_AUTO,
HA_OPTION_DISABLED, HA_OPTION_MASTER, HA_OPTION_STANDBY, HA_OPTION_HOT_STANDBY); }
+        protected final static String HA_OPTION_HOT_BACKUP = "hot_backup";
+        static { Enums.checkAllEnumeratedIgnoreCase(HighAvailabilityMode.class, HA_OPTION_AUTO,
HA_OPTION_DISABLED, HA_OPTION_MASTER, HA_OPTION_STANDBY, HA_OPTION_HOT_STANDBY, HA_OPTION_HOT_BACKUP);
}
         
-        @Option(name = { HA_OPTION }, allowedValues = { HA_OPTION_DISABLED, HA_OPTION_AUTO,
HA_OPTION_MASTER, HA_OPTION_STANDBY, HA_OPTION_HOT_STANDBY },
+        @Option(name = { HA_OPTION }, allowedValues = { HA_OPTION_DISABLED, HA_OPTION_AUTO,
HA_OPTION_MASTER, HA_OPTION_STANDBY, HA_OPTION_HOT_STANDBY, HA_OPTION_HOT_BACKUP },
                 title = "high availability mode",
                 description =
                         "The high availability mode. Possible values are: \n"+
                         "disabled: management node works in isolation - will not cooperate
with any other standby/master nodes in management plane; \n"+
                         "auto: will look for other management nodes, and will allocate itself
as standby or master based on other nodes' states; \n"+
                         "master: will startup as master - if there is already a master then
fails immediately; \n"+
-                        "standby: will start up as lukewarm standby - if there is not already
a master then fails immediately; \n"+
-                        "hot_standby: will start up as hot standby - if there is not already
a master then fails immediately")
+                        "standby: will start up as lukewarm standby with no state - if there
is not already a master then fails immediately, "
+                        + "and if there is a master which subsequently fails, this node can
promote itself; \n"+
+                        "hot_standby: will start up as hot standby in read-only mode - if
there is not already a master then fails immediately, "
+                        + "and if there is a master which subseuqently fails, this node can
promote itself; \n"+
+                        "hot_backup: will start up as hot backup in read-only mode - no master
is required, and this node will not become a master"
+                        )
         public String highAvailability = HA_OPTION_AUTO;
 
         @VisibleForTesting
@@ -472,7 +477,10 @@ public class Main extends AbstractMain {
                     if (highAvailabilityMode.get() == HighAvailabilityMode.AUTO)
                         return HighAvailabilityMode.DISABLED;
                     throw new FatalConfigurationRuntimeException("Cannot specify highAvailability
when persistence is disabled");
-                } else if (persistMode == PersistMode.CLEAN && (highAvailabilityMode.get()
== HighAvailabilityMode.STANDBY || highAvailabilityMode.get() == HighAvailabilityMode.HOT_STANDBY))
{
+                } else if (persistMode == PersistMode.CLEAN && 
+                        (highAvailabilityMode.get() == HighAvailabilityMode.STANDBY 
+                        || highAvailabilityMode.get() == HighAvailabilityMode.HOT_STANDBY
+                        || highAvailabilityMode.get() == HighAvailabilityMode.HOT_BACKUP))
{
                     throw new FatalConfigurationRuntimeException("Cannot specify highAvailability
"+highAvailabilityMode.get()+" when persistence is CLEAN");
                 }
             }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9dd1a957/usage/launcher/src/main/java/brooklyn/launcher/BrooklynLauncher.java
----------------------------------------------------------------------
diff --git a/usage/launcher/src/main/java/brooklyn/launcher/BrooklynLauncher.java b/usage/launcher/src/main/java/brooklyn/launcher/BrooklynLauncher.java
index 00162a8..976db4c 100644
--- a/usage/launcher/src/main/java/brooklyn/launcher/BrooklynLauncher.java
+++ b/usage/launcher/src/main/java/brooklyn/launcher/BrooklynLauncher.java
@@ -160,8 +160,9 @@ public class BrooklynLauncher {
     private String persistenceDir;
     private String persistenceLocation;
     private Duration persistPeriod = Duration.ONE_SECOND;
-    private Duration haHeartbeatTimeout = Duration.THIRTY_SECONDS;
-    private Duration haHeartbeatPeriod = Duration.ONE_SECOND;
+    // these default values come from config in HighAvailablilityManagerImpl
+    private Duration haHeartbeatTimeoutOverride = null;
+    private Duration haHeartbeatPeriodOverride = null;
     
     private volatile BrooklynWebServer webServer;
     private CampPlatform campPlatform;
@@ -433,7 +434,7 @@ public class BrooklynLauncher {
     }
 
     public BrooklynLauncher haHeartbeatTimeout(Duration val) {
-        this.haHeartbeatTimeout = val;
+        this.haHeartbeatTimeoutOverride = val;
         return this;
     }
 
@@ -446,7 +447,7 @@ public class BrooklynLauncher {
      * Controls both the frequency of heartbeats, and the frequency of checking the health
of other nodes.
      */
     public BrooklynLauncher haHeartbeatPeriod(Duration val) {
-        this.haHeartbeatPeriod = val;
+        this.haHeartbeatPeriodOverride = val;
         return this;
     }
 
@@ -770,8 +771,8 @@ public class BrooklynLauncher {
             ManagementPlaneSyncRecordPersister persister =
                 new ManagementPlaneSyncRecordPersisterToObjectStore(managementContext,
                     objectStore, managementContext.getCatalog().getRootClassLoader());
-            ((HighAvailabilityManagerImpl)haManager).setHeartbeatTimeout(haHeartbeatTimeout);
-            ((HighAvailabilityManagerImpl)haManager).setPollPeriod(haHeartbeatPeriod);
+            ((HighAvailabilityManagerImpl)haManager).setHeartbeatTimeout(haHeartbeatTimeoutOverride);
+            ((HighAvailabilityManagerImpl)haManager).setPollPeriod(haHeartbeatPeriodOverride);
             haManager.setPersister(persister);
         }
     }
@@ -790,19 +791,20 @@ public class BrooklynLauncher {
             // Let the HA manager decide when objectstore.prepare and rebindmgr.rebind need
to be called 
             // (based on whether other nodes in plane are already running).
             
-            HighAvailabilityMode startMode;
+            HighAvailabilityMode startMode=null;
             switch (highAvailabilityMode) {
                 case AUTO:
                 case MASTER:
                 case STANDBY:
                 case HOT_STANDBY:
+                case HOT_BACKUP:
                     startMode = highAvailabilityMode;
                     break;
                 case DISABLED:
                     throw new IllegalStateException("Unexpected code-branch for high availability
mode "+highAvailabilityMode);
-                default:       
-                    throw new IllegalStateException("Unexpected high availability mode "+highAvailabilityMode);
             }
+            if (startMode==null)
+                throw new IllegalStateException("Unexpected high availability mode "+highAvailabilityMode);
             
             LOG.debug("Management node (with HA) starting");
             HighAvailabilityManager haManager = managementContext.getHighAvailabilityManager();


Mime
View raw message