brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [3/9] git commit: support changing mode to disabled, ensuring things really are purged from memory; and allow HA tests based on RebindTestUtils/Fixture
Date Fri, 17 Oct 2014 10:24:13 GMT
support changing mode to disabled, ensuring things really are purged from memory; and allow
HA tests based on RebindTestUtils/Fixture


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/5966f3b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/5966f3b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/5966f3b1

Branch: refs/heads/master
Commit: 5966f3b15d2781b6fac59ce944948199b29cd1a1
Parents: af1925e
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Wed Oct 15 23:18:03 2014 +0100
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Thu Oct 16 17:44:39 2014 +0100

----------------------------------------------------------------------
 .../brooklyn/catalog/internal/CatalogDto.java   | 19 ++++--
 .../entity/rebind/RebindManagerImpl.java        |  3 +
 .../location/basic/AbstractLocation.java        | 10 ++-
 .../ha/HighAvailabilityManagerImpl.java         | 71 ++++++++++++++++----
 .../management/internal/LocalEntityManager.java |  6 ++
 .../internal/LocalLocationManager.java          | 10 ++-
 .../brooklyn/entity/rebind/RebindTestUtils.java |  3 +
 .../brooklyn/management/ha/HotStandbyTest.java  | 33 +++++++++
 8 files changed, 132 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/catalog/internal/CatalogDto.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/catalog/internal/CatalogDto.java b/core/src/main/java/brooklyn/catalog/internal/CatalogDto.java
index 47269dc..c4e08b1 100644
--- a/core/src/main/java/brooklyn/catalog/internal/CatalogDto.java
+++ b/core/src/main/java/brooklyn/catalog/internal/CatalogDto.java
@@ -26,17 +26,17 @@ import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.annotations.Beta;
-import com.google.common.base.Objects;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-
 import brooklyn.catalog.CatalogItem;
 import brooklyn.util.ResourceUtils;
 import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.exceptions.PropagatedRuntimeException;
 import brooklyn.util.stream.Streams;
 
+import com.google.common.annotations.Beta;
+import com.google.common.base.Objects;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Lists;
+
 @Beta
 public class CatalogDto {
 
@@ -103,6 +103,13 @@ public class CatalogDto {
         return result;
     }
 
+    /** Used when caller wishes to create an explicitly empty catalog */
+    public static CatalogDto newEmptyInstance(String optionalContentsDescription) {
+        CatalogDto result = new CatalogDto();
+        if (optionalContentsDescription!=null) result.contentsDescription = optionalContentsDescription;
+        return result;
+    }
+
     public static CatalogDto newLinkedInstance(String url) {
         CatalogDto result = new CatalogDto();
         result.contentsDescription = url;
@@ -110,7 +117,7 @@ public class CatalogDto {
         return result;
     }
 
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings({ "unchecked", "rawtypes" })
     public static CatalogDto newDtoFromCatalogItems(Collection<CatalogItem<?, ?>>
entries) {
         CatalogDto result = new CatalogDto();
         // Weird casts because compiler does not seem to like

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
index 35be729..7a01db9 100644
--- a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
+++ b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
@@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -47,6 +48,7 @@ import brooklyn.entity.Feed;
 import brooklyn.entity.basic.AbstractApplication;
 import brooklyn.entity.basic.AbstractEntity;
 import brooklyn.entity.basic.ConfigKeys;
+import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.proxying.InternalEntityFactory;
 import brooklyn.entity.proxying.InternalFactory;
@@ -870,6 +872,7 @@ public class RebindManagerImpl implements RebindManager {
                         LOG.debug(message, catalogLoadMode);
                     }
                 }
+                // TODO destroy old (as above)
             } else {
                 LOG.debug("RebindManager not resetting catalog because catalog persistence
is disabled");
             }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/location/basic/AbstractLocation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/location/basic/AbstractLocation.java b/core/src/main/java/brooklyn/location/basic/AbstractLocation.java
index ee12da9..e619c57 100644
--- a/core/src/main/java/brooklyn/location/basic/AbstractLocation.java
+++ b/core/src/main/java/brooklyn/location/basic/AbstractLocation.java
@@ -312,6 +312,10 @@ public abstract class AbstractLocation extends AbstractBrooklynObject
implements
 
     @Override
     public void setParent(Location newParent) {
+        setParent(newParent, true);
+    }
+    
+    public void setParent(Location newParent, boolean updateChildListInOldParent) {
         if (newParent == this) {
             throw new IllegalArgumentException("Location cannot be its own parent: "+this);
         }
@@ -323,11 +327,13 @@ public abstract class AbstractLocation extends AbstractBrooklynObject
implements
         if (parent.get() != null) {
             Location oldParent = parent.get();
             parent.set(null);
-            ((AbstractLocation)oldParent).removeChild(this); // FIXME Nasty cast
+            if (updateChildListInOldParent)
+                ((AbstractLocation)oldParent).removeChild(this);
         }
         if (newParent != null) {
             parent.set(newParent);
-            ((AbstractLocation)parent.get()).addChild(this); // FIXME Nasty cast
+            if (updateChildListInOldParent)
+                ((AbstractLocation)parent.get()).addChild(this);
         }
         
         onChanged();

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
index fb339d4..a432109 100644
--- a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
+++ b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
@@ -33,18 +33,24 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import brooklyn.BrooklynVersion;
+import brooklyn.catalog.internal.BasicBrooklynCatalog;
+import brooklyn.catalog.internal.CatalogDto;
 import brooklyn.entity.Application;
 import brooklyn.entity.Entity;
-import brooklyn.entity.basic.Entities;
+import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.rebind.RebindManager;
 import brooklyn.entity.rebind.plane.dto.BasicManagementNodeSyncRecord;
 import brooklyn.entity.rebind.plane.dto.ManagementPlaneSyncRecordImpl;
 import brooklyn.entity.rebind.plane.dto.ManagementPlaneSyncRecordImpl.Builder;
 import brooklyn.internal.BrooklynFeatureEnablement;
+import brooklyn.location.Location;
 import brooklyn.management.Task;
 import brooklyn.management.ha.BasicMasterChooser.AlphabeticMasterChooser;
 import brooklyn.management.ha.ManagementPlaneSyncRecordPersister.Delta;
+import brooklyn.management.internal.LocalEntityManager;
+import brooklyn.management.internal.LocationManagerInternal;
 import brooklyn.management.internal.ManagementContextInternal;
+import brooklyn.management.internal.ManagementTransitionInfo.ManagementTransitionMode;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.collections.MutableSet;
 import brooklyn.util.exceptions.Exceptions;
@@ -222,8 +228,17 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
     @VisibleForTesting
     @Beta
     public void changeMode(HighAvailabilityMode startMode, boolean preventElectionOnExplicitStandbyMode,
boolean failOnExplicitStandbyModeIfNoMaster) {
-        if (!running)
-            throw new IllegalStateException("Can only change mode when already running; invoke
'start' first");
+        if (!running) {
+            // if was not running then start as disabled mode, then proceed as normal
+            LOG.info("HA changing mode to "+startMode+" from "+nodeState+" when not running,
forcing an intermediate start as DISABLED then will convert to "+startMode);
+            start(HighAvailabilityMode.DISABLED);
+        }
+        if (getNodeState()==ManagementNodeState.FAILED || getNodeState()==ManagementNodeState.INITIALIZING)
{
+            if (startMode!=HighAvailabilityMode.DISABLED) {
+                // if coming from FAILED (or INITIALIZING because we skipped start call)
then treat as cold standby
+                nodeState = ManagementNodeState.STANDBY; 
+            }
+        }
         
         ownNodeId = managementContext.getManagementNodeId();
         // TODO Small race in that we first check, and then we'll do checkMaster() on first
poll,
@@ -240,11 +255,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             switch (startMode) {
             case MASTER:
             case AUTO:
-                // no action needed
+            case DISABLED:
+                // no action needed, will do anything necessary below
                 break;
             case HOT_STANDBY: demoteToStandby(true); break;
             case STANDBY: demoteToStandby(false); break;
-            case DISABLED: demoteToFailed(); break;
             default:
                 throw new IllegalStateException("Unexpected high availability mode "+startMode+"
requested for "+this);
             }
@@ -294,7 +309,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             if (getNodeState().toString().equals(startMode.toString()))
                 message += "explicitly requested";
             else if (startMode==HighAvailabilityMode.HOT_STANDBY && getNodeState()==ManagementNodeState.STANDBY)
-                message += "caller requested "+startMode+", will attempt rebind directly";
+                message += "caller requested "+startMode+", will attempt rebind for HOT_STANDBY
next";
             else
                 message += "caller requested "+startMode;
             
@@ -310,6 +325,10 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             }
             LOG.info(message);
             break;
+        case DISABLED:
+            // safe just to run even if we weren't master
+            demoteToFailed();
+            break;
         default:
             throw new IllegalStateException("Unexpected high availability mode "+startMode+"
requested for "+this);
         }
@@ -643,6 +662,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                 // could just promote the standby items; but for now we stop the old read-only
and re-load them, to make sure nothing has been missed
                 // TODO ideally there'd be an incremental rebind as well as an incremental
persist
                 managementContext.getRebindManager().stopReadOnly();
+                clearManagedItems(ManagementTransitionMode.REBINDING_DESTROYED);
             }
             managementContext.getRebindManager().rebind(managementContext.getCatalog().getRootClassLoader(),
null, nodeState);
         } catch (Exception e) {
@@ -654,8 +674,9 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
     }
     
     protected void demoteToFailed() {
+        ManagementTransitionMode mode = (nodeState == ManagementNodeState.MASTER ? ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY
: ManagementTransitionMode.REBINDING_DESTROYED);
         nodeState = ManagementNodeState.FAILED;
-        onDemotionStopTasks();
+        onDemotionStopItems(mode);
         nodeStateTransitionComplete = true;
         publishDemotionFromMaster();
     }
@@ -665,10 +686,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             LOG.warn("Ignoring demote-from-master request, as HighAvailabilityManager is
no longer running");
             return;
         }
+        ManagementTransitionMode mode = (nodeState == ManagementNodeState.MASTER ? ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY
: ManagementTransitionMode.REBINDING_DESTROYED);
 
         nodeStateTransitionComplete = false;
         nodeState = ManagementNodeState.STANDBY;
-        onDemotionStopTasks();
+        onDemotionStopItems(mode);
         nodeStateTransitionComplete = true;
         publishDemotionFromMaster();
         
@@ -680,11 +702,12 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         }
     }
     
-    protected void onDemotionStopTasks() {
+    protected void onDemotionStopItems(ManagementTransitionMode mode) {
+        // stop persistence and remove all apps etc
         managementContext.getRebindManager().stopPersistence();
-        for (Application app: managementContext.getApplications())
-            Entities.unmanage(app);
-        // let's try forcibly interrupting tasks on managed entities
+        clearManagedItems(mode);
+        
+        // try forcibly interrupting tasks on managed entities
         Collection<Exception> exceptions = MutableSet.of();
         int tasks = 0;
         LOG.debug("Cancelling tasks on demotion");
@@ -714,6 +737,30 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
             LOG.info("Cancelled "+tasks+" tasks on demotion");
     }
 
+    /** clears all managed items from the management context; same items destroyed as in
the course of a rebind cycle */
+    protected void clearManagedItems(ManagementTransitionMode mode) {
+        for (Application app: managementContext.getApplications()) {
+            if (((EntityInternal)app).getManagementSupport().isDeployed()) {
+                ((EntityInternal)app).getManagementContext().getEntityManager().unmanage(app);
+            }
+        }
+        // for normal management, call above will remove; for read-only, etc, let's do what's
below:
+        for (Entity entity: managementContext.getEntityManager().getEntities()) {
+            ((LocalEntityManager)managementContext.getEntityManager()).unmanage(entity, mode);
+        }
+    
+        // again, for locations, call unmanage on parents first
+        for (Location loc: managementContext.getLocationManager().getLocations()) {
+            if (loc.getParent()==null)
+                ((LocationManagerInternal)managementContext.getLocationManager()).unmanage(loc,
mode);
+        }
+        for (Location loc: managementContext.getLocationManager().getLocations()) {
+            ((LocationManagerInternal)managementContext.getLocationManager()).unmanage(loc,
mode);
+        }
+        
+        ((BasicBrooklynCatalog)managementContext.getCatalog()).reset(CatalogDto.newEmptyInstance("<reset-by-ha-status-change>"));
+    }
+    
     /** starts hot standby, in foreground; the caller is responsible for publishing health
afterwards.
      * @return whether hot standby was possible (if not, errors should be stored elsewhere)
*/
     protected boolean attemptHotStandby() {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/management/internal/LocalEntityManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/LocalEntityManager.java b/core/src/main/java/brooklyn/management/internal/LocalEntityManager.java
index adc9839..1778e99 100644
--- a/core/src/main/java/brooklyn/management/internal/LocalEntityManager.java
+++ b/core/src/main/java/brooklyn/management/internal/LocalEntityManager.java
@@ -384,6 +384,12 @@ public class LocalEntityManager implements EntityManagerInternal {
         } else {
             log.warn("Invalid mode for unmanage: "+mode+" on "+e+" (ignoring)");
         }
+        
+        preRegisteredEntitiesById.remove(e.getId());
+        preManagedEntitiesById.remove(e.getId());
+        entityProxiesById.remove(e.getId());
+        entitiesById.remove(e.getId());
+        entityModesById.remove(e.getId());
     }
     
     /**

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/main/java/brooklyn/management/internal/LocalLocationManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/LocalLocationManager.java b/core/src/main/java/brooklyn/management/internal/LocalLocationManager.java
index 3fe711e..b5f9bd4 100644
--- a/core/src/main/java/brooklyn/management/internal/LocalLocationManager.java
+++ b/core/src/main/java/brooklyn/management/internal/LocalLocationManager.java
@@ -271,8 +271,8 @@ public class LocalLocationManager implements LocationManagerInternal {
                     log.warn("Should not be unmanaging "+loc+" in mode "+mode+"; ignoring");
             }
 
-        } else if (mode==ManagementTransitionMode.REBINDING_DESTROYED) {
-            // we are unmanaging an instance (secondary) for which the primary has been destroyed
elsewhere
+        } else if (mode==ManagementTransitionMode.REBINDING_DESTROYED || mode==ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY)
{
+            // we are unmanaging an instance whose primary management is elsewhere (either
we were secondary, or we are being demoted)
             unmanageNonRecursive(loc);
             managementContext.getRebindManager().getChangeListener().onUnmanaged(loc);
             if (managementContext.gc != null) managementContext.gc.onUnmanaged(loc);
@@ -304,6 +304,10 @@ public class LocalLocationManager implements LocationManagerInternal
{
             log.warn("Invalid mode for unmanage: "+mode+" on "+loc+" (ignoring)");
         }
         
+        locationsById.remove(loc.getId());
+        preRegisteredLocationsById.remove(loc.getId());
+        locationModesById.remove(loc.getId());
+        locationTypes.remove(loc.getId());
     }
     
     /**
@@ -363,7 +367,7 @@ public class LocalLocationManager implements LocationManagerInternal {
      * Returns true if the location has been removed from management; if it was not previously
managed (anything else throws exception) 
      */
     private synchronized boolean unmanageNonRecursive(Location loc) {
-        loc.setParent(null);
+        ((AbstractLocation)loc).setParent(null, false);
         Object old = locationsById.remove(loc.getId());
         locationTypes.remove(loc.getId());
         locationModesById.remove(loc.getId());

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/test/java/brooklyn/entity/rebind/RebindTestUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/RebindTestUtils.java b/core/src/test/java/brooklyn/entity/rebind/RebindTestUtils.java
index 61e130a..bef1763 100644
--- a/core/src/test/java/brooklyn/entity/rebind/RebindTestUtils.java
+++ b/core/src/test/java/brooklyn/entity/rebind/RebindTestUtils.java
@@ -46,6 +46,7 @@ import brooklyn.location.Location;
 import brooklyn.management.ManagementContext;
 import brooklyn.management.ha.HighAvailabilityMode;
 import brooklyn.management.ha.ManagementNodeState;
+import brooklyn.management.ha.ManagementPlaneSyncRecordPersisterToObjectStore;
 import brooklyn.management.internal.LocalManagementContext;
 import brooklyn.management.internal.ManagementContextInternal;
 import brooklyn.mementos.BrooklynMemento;
@@ -206,6 +207,8 @@ public class RebindTestUtils {
                     classLoader);
             ((RebindManagerImpl) unstarted.getRebindManager()).setPeriodicPersistPeriod(persistPeriod);
             unstarted.getRebindManager().setPersister(newPersister, PersistenceExceptionHandlerImpl.builder().build());
+            // set the HA persister, in case any children want to use HA
+            unstarted.getHighAvailabilityManager().setPersister(new ManagementPlaneSyncRecordPersisterToObjectStore(unstarted,
objectStore, classLoader));
             return unstarted;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/5966f3b1/core/src/test/java/brooklyn/management/ha/HotStandbyTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/management/ha/HotStandbyTest.java b/core/src/test/java/brooklyn/management/ha/HotStandbyTest.java
index a34f26b..67989fd 100644
--- a/core/src/test/java/brooklyn/management/ha/HotStandbyTest.java
+++ b/core/src/test/java/brooklyn/management/ha/HotStandbyTest.java
@@ -45,6 +45,8 @@ import brooklyn.entity.rebind.persister.ListeningObjectStore;
 import brooklyn.entity.rebind.persister.PersistMode;
 import brooklyn.entity.rebind.persister.PersistenceObjectStore;
 import brooklyn.location.Location;
+import brooklyn.location.LocationSpec;
+import brooklyn.location.basic.LocalhostMachineProvisioningLocation.LocalhostMachine;
 import brooklyn.management.internal.AbstractManagementContext;
 import brooklyn.management.internal.ManagementContextInternal;
 import brooklyn.test.EntityTestUtils;
@@ -560,5 +562,36 @@ public class HotStandbyTest {
     public void testChangeModeManyTimes() throws Exception {
         testChangeMode();
     }
+
+    @Test
+    public void testChangeModeToDisabledAndBack() throws Exception {
+        HaMgmtNode n1 = createMaster(Duration.PRACTICALLY_FOREVER);
+        n1.mgmt.getLocationManager().createLocation(LocationSpec.create(LocalhostMachine.class));
+        @SuppressWarnings("unused")
+        TestApplication app = createFirstAppAndPersist(n1);
+        
+        HaMgmtNode n2 = createHotStandby(Duration.PRACTICALLY_FOREVER);
+        
+        // disabled n1 allows n2 to become master when next we tell it to check
+        n1.ha.changeMode(HighAvailabilityMode.DISABLED);
+        n2.ha.changeMode(HighAvailabilityMode.AUTO);
+        assertMaster(n2);
+        assertEquals(n1.ha.getNodeState(), ManagementNodeState.FAILED);
+        Assert.assertTrue(n1.mgmt.getApplications().isEmpty(), "n1 should have had no apps;
instead had: "+n1.mgmt.getApplications());
+        Assert.assertTrue(n1.mgmt.getEntityManager().getEntities().isEmpty(), "n1 should
have had no entities; instead had: "+n1.mgmt.getEntityManager().getEntities());
+        Assert.assertTrue(n1.mgmt.getLocationManager().getLocations().isEmpty(), "n1 should
have had no locations; instead had: "+n1.mgmt.getLocationManager().getLocations());
+        
+        // we can now change n1 back to hot_standby
+        n1.ha.changeMode(HighAvailabilityMode.HOT_STANDBY);
+        assertHotStandby(n1);
+        // and it sees apps
+        Assert.assertFalse(n1.mgmt.getApplications().isEmpty(), "n1 should have had apps
now");
+        Assert.assertFalse(n1.mgmt.getApplications().isEmpty(), "n1 should have had locations
now");
+        // and if n2 is disabled, n1 promotes
+        n2.ha.changeMode(HighAvailabilityMode.DISABLED);
+        n1.ha.changeMode(HighAvailabilityMode.AUTO);
+        assertMaster(n1);
+        assertEquals(n2.ha.getNodeState(), ManagementNodeState.FAILED);
+    }
     
 }


Mime
View raw message